├── .gitignore
├── requirements.txt
├── dbmixin.py
├── README.md
├── config.py
├── utils.py
├── app.py
├── templates
    └── index.html
├── spider.py
└── proxy_list.txt


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask==1.0.2
2 | gevent==1.1.2
3 | lxml==4.6.2
4 | pymongo==2.8
5 | requests==2.22.0
6 | 


--------------------------------------------------------------------------------
/dbmixin.py:
--------------------------------------------------------------------------------
 1 | # coding: utf8
 2 | 
 3 | """ 单例mongo客户端 """
 4 | 
 5 | from pymongo import MongoClient
 6 | 
 7 | from config import DB_URI, DB_NAME
 8 | 
 9 | 
10 | class DBMixin(object):
11 | 
12 |     """ mongodb客户端 """
13 | 
14 |     _client = None
15 | 
16 |     @property
17 |     def client(self):
18 |         if not self._client:
19 |             self._client = MongoClient(DB_URI)
20 |         return self._client
21 | 
22 |     @property
23 |     def db(self):
24 |         return self.client[DB_NAME]
25 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # 说明
 2 | 用于爬取`豆瓣小组`的爬虫。<br/>
 3 | 此爬虫我主要用于了爬取`豆瓣租房小组`的帖子，支持`关键字搜索`以及`发帖`、`更新时间排序`。 
 4 | 
 5 | # 依赖
 6 | - `gevent`
 7 | - `pymongo`
 8 | - `requests`
 9 | - `lxml`
10 | - `Flask`
11 | -  `boostrap`
12 | <br/>
13 | 具体版本参见`requirements.txt`<br/>
14 | 
15 | # 特别说明
16 | - 由于豆瓣有防抓机制，故此爬虫使用了代理爬取，防止被封IP。<br/>
17 | - 可从网上收集代理IP，放在项目路径下`proxy_list.txt`。
18 | - 每个一行，程序会自动加载，且可以自动定时加载新代理。<br/>
19 | - 或者参考我的[代理采集器](https://github.com/kaito-kidd/proxy-fetcher)，自动采集代理。
20 | - 如果程序运行发现总是出现超时或者403，请更换`proxy_list.txt`下的代理。
21 | 
22 | # 使用
23 | - 安装`MongoDB`，具体参考安装文档。
24 | - 建议使用`virtualenv`环境<br/>
25 |     `pip install -r requirements.txt`
26 | - 启动爬虫<br/>
27 |     `nohup python spider.py >> douban_spider.log &`
28 | - 启动web服务<br/>
29 |     `nohup python app.py >> app.log &`
30 | - 查看页面<br/>
31 |     `http://localhost:5000`
32 | 
33 | # 配置
34 | 参数配置见`config.py`，例如`MongoDB地址`、`并发数`、`爬取页数`等。
35 | 


--------------------------------------------------------------------------------
/config.py:
--------------------------------------------------------------------------------
 1 | # coding: utf8
 2 | 
 3 | """配置
 4 | """
 5 | 
 6 | # User-Agent
 7 | USER_AGENT = (
 8 |     "Mozilla/5.0 (Windows NT 6.3; WOW64) "
 9 |     "AppleWebKit/537.36 (KHTML, like Gecko) "
10 |     "Chrome/40.0.2214.93 Safari/537.36"
11 | )
12 | 
13 | # mongo config
14 | DB_URI = "mongodb://127.0.0.1:27017"
15 | # DB_URI = "mongodb://192.168.6.7:27017"
16 | DB_NAME = "douban_group"
17 | 
18 | # 豆瓣小组URL
19 | GROUP_LIST = [
20 |     # 北京租房豆瓣
21 |     "http://www.douban.com/group/26926/",
22 |     # 北京租房（非中介）
23 |     "http://www.douban.com/group/279962/",
24 |     # 北京租房房东联盟(中介勿扰)
25 |     "http://www.douban.com/group/257523/",
26 |     # 北京租房
27 |     "http://www.douban.com/group/beijingzufang/",
28 |     # 北京租房小组
29 |     "http://www.douban.com/group/xiaotanzi/",
30 |     # 北京无中介租房
31 |     "http://www.douban.com/group/zhufang/",
32 |     # 北漂爱合租
33 |     "http://www.douban.com/group/aihezu/",
34 |     # 北京同志们来租房
35 |     "http://www.douban.com/group/325060/",
36 |     # 北京个人租房
37 |     "http://www.douban.com/group/opking/",
38 |     # 北京租房小组!
39 |     "http://www.douban.com/group/374051/",
40 | ]
41 | 
42 | # 后缀
43 | GROUP_SUFFIX = "discussion?start=%d"
44 | 
45 | # 抓取前多少页
46 | MAX_PAGE = 5
47 | 
48 | # 匹配规则
49 | RULES = {
50 |     # 每个帖子项
51 |     "topic_item": "//table[@class='olt']/tr",
52 |     "url_list": "//table[@class='olt']/tr/td[@class='title']/a/@href",
53 |     # 列表元素
54 |     "title": "td[@class='title']/a/@title",
55 |     "author": "td[@nowrap='nowrap'][1]/a/text()",
56 |     "reply": "td[@nowrap='nowrap'][2]/text()",
57 |     "last_reply_time": "td[@class='time']/text()",
58 |     "url": "td[@class='title']/a/@href",
59 |     # 帖子详情
60 |     "detail_title_sm": "//td[@class='tablecc']/text()",
61 |     # 完整标题
62 |     "detail_title_lg": "//div[@id='content']/h1/text()",
63 |     "create_time": "//span[@class='color-green']/text()",
64 |     "detail_author": "//span[@class='from']/a/text()",
65 |     "content": "//div[@class='topic-content']/p/text()",
66 | }
67 | 
68 | # 并发数
69 | POOL_SIZE = 20
70 | 
71 | # 监控周期(秒),默认10分钟
72 | WATCH_INTERVAL = 10 * 60
73 | 
74 | # 重载代理周期
75 | PROXY_INTERVAL = 30 * 60
76 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | import time
 5 | import logging
 6 | import random
 7 | 
 8 | 
 9 | class Timer(object):
10 |     """定时器，定时执行指定的函数
11 | 
12 |     """
13 | 
14 |     def __init__(self, start, interval):
15 |         """
16 |         @start, int, 延迟执行的秒数
17 |         @interval, int, 每次执行的间隔秒数
18 |         """
19 |         self.start = start
20 |         self.interval = interval
21 | 
22 |     def run(self, func, *args, **kwargs):
23 |         """运行定时器
24 | 
25 |         @func, callable, 要执行的函数
26 |         """
27 |         time.sleep(self.start)
28 |         while True:
29 |             func(*args, **kwargs)
30 |             time.sleep(self.interval)
31 | 
32 | 
33 | class ProxyManager(object):
34 |     """代理管理器
35 |     """
36 | 
37 |     def __init__(self, proxies_or_path, interval_per_ip=0, is_single=False):
38 |         '''
39 |         @proxies_or_path, basestring or list, 代理path或列表
40 |         @interval_per_ip, int, 每个ip调用最小间隔
41 |         @is_single, bool, 是否启用单点代理,例如使用squid
42 |         '''
43 |         self.proxies_or_path = proxies_or_path
44 |         self.host_time_map = {}
45 |         self.interval = interval_per_ip
46 |         self.is_single = is_single
47 |         self.init_proxies(self.proxies_or_path)
48 | 
49 |     def init_proxies(self, proxies_or_path):
50 |         '''初始化代理列表
51 | 
52 |         @proxies_or_path, list or basestring
53 |         '''
54 |         if isinstance(proxies_or_path, basestring):
55 |             if self.is_single:
56 |                 self.proxies = proxies_or_path
57 |             else:
58 |                 with open(proxies_or_path) as f:
59 |                     self.proxies = f.readlines()
60 |         else:
61 |             self.proxies = proxies_or_path
62 | 
63 |     def reload_proxies(self):
64 |         '''重新加载代理，proxies_or_path必须是文件路径
65 |         '''
66 |         if not isinstance(self.proxies_or_path, basestring):
67 |             raise TypeError("proxies_or_path type is invalid!")
68 |         if self.is_single:
69 |             raise TypeError("is_single must be False!")
70 |         with open(self.proxies_or_path) as f:
71 |             self.proxies = f.readlines()
72 |         logging.info("reload %s proxies ...", len(self.proxies))
73 | 
74 |     def get_proxy(self):
75 |         '''获取一个可用代理
76 | 
77 |         如果代理使用过于频繁会阻塞，以防止服务器屏蔽
78 |         '''
79 |         # 如果使用单点代理,直接返回
80 |         if self.is_single:
81 |             return self.proxies
82 |         proxy = self.proxies[random.randint(0, len(self.proxies) - 1)].strip()
83 |         host, _ = proxy.split(':')
84 |         latest_time = self.host_time_map.get(host, 0)
85 |         interval = time.time() - latest_time
86 |         if interval < self.interval:
87 |             logging.info("%s waiting", proxy)
88 |             time.sleep(self.interval)
89 |         self.host_time_map[host] = time.time()
90 |         return "http://%s" % proxy.strip()
91 | 


--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
  1 | # coding: utf8
  2 | 
  3 | import re
  4 | 
  5 | from flask import Flask, request, render_template
  6 | from dbmixin import DBMixin
  7 | 
  8 | app = Flask(__name__)
  9 | app.debug = True
 10 | 
 11 | # db & collection
 12 | db = DBMixin().db
 13 | topic_collection = db.result_topic
 14 | page_collection = db.result_page
 15 | 
 16 | 
 17 | @app.route("/", methods=["GET", "POST"])
 18 | def index():
 19 |     """index
 20 |     """
 21 |     cpage = int(request.form.get("cpage", 1))
 22 |     psize = int(request.form.get("psize", 100))
 23 |     keyword = request.form.get("keyword", "")
 24 |     is_order_update_time = int(request.form.get(
 25 |         "is_order_update_time", 0))
 26 |     # 按更新时间排序
 27 |     if is_order_update_time:
 28 |         orderby = "last_update_time"
 29 |         collection = page_collection
 30 |     else:
 31 |         orderby = "create_time"
 32 |         collection = topic_collection
 33 |     where = {}
 34 |     if keyword:
 35 |         regx = re.compile(".*%s.*" % keyword, re.IGNORECASE)
 36 |         where["title"] = {"$regex": regx}
 37 |     # count
 38 |     count = collection.find(where).count()
 39 |     # page
 40 |     page = Page(cpage, psize, count)
 41 |     # data
 42 |     topics = collection.find(where) \
 43 |         .sort([(orderby, -1)]).skip(page.start) \
 44 |         .limit(page.end)
 45 |     return render_template(
 46 |         "index.html",
 47 |         topics=topics,
 48 |         is_order_update_time=is_order_update_time,
 49 |         keyword=keyword,
 50 |         count=count,
 51 |         page=page)
 52 | 
 53 | 
 54 | class Page(object):
 55 | 
 56 |     """ page info"""
 57 | 
 58 |     def __init__(self, cpage, psize, count):
 59 |         self.cpage = cpage
 60 |         self.psize = psize
 61 |         self.count = count
 62 |         self.max_page = (count + psize - 1) / psize
 63 |         self.max_page = max(self.max_page, 1)
 64 |         self.cpage = min(self.cpage, self.max_page)
 65 | 
 66 |         self.start = (self.cpage - 1) * psize
 67 |         self.start = 0 if self.start < 0 else self.start
 68 |         self.end = psize
 69 | 
 70 |         # 初始化页码
 71 |         self.init_page_num(self.cpage, self.max_page)
 72 | 
 73 |     def init_page_num(self, cpage, max_page):
 74 |         '''初始化页面页码
 75 | 
 76 |         @cpage, int, 当前页码
 77 |         @max_page, int, 最大页数
 78 |         '''
 79 |         begin_page, end_page = 1, 1
 80 |         # 总页码小于15
 81 |         if max_page < 15:
 82 |             # 开始:1
 83 |             begin_page = 1
 84 |             # 结束:总页码
 85 |             end_page = max_page
 86 |         else:
 87 |             # 保证显示15个
 88 |             begin_page = cpage - 7
 89 |             end_page = cpage + 7
 90 |             if begin_page < 1:
 91 |                 begin_page = 1
 92 |                 end_page = 15
 93 |             elif end_page > max_page:
 94 |                 end_page = max_page
 95 |                 begin_page = max_page - 15
 96 |         self.begin_page = begin_page
 97 |         self.end_page = end_page
 98 | 
 99 | 
100 | if __name__ == "__main__":
101 |     app.run(host="0.0.0.0")
102 | 


--------------------------------------------------------------------------------
/templates/index.html:
--------------------------------------------------------------------------------
  1 | <!doctype html>
  2 | <html>
  3 |     <head>
  4 |         <meta charset="utf-8">
  5 |         <meta http-equiv="X-UA-Compatible" content="IE=edge">
  6 |         <meta name="viewport" content="width=device-width, initial-scale=1">
  7 |         <title>豆瓣租房搜索</title>
  8 |         <link rel="stylesheet" href="http://cdn.bootcss.com/bootstrap/3.3.4/css/bootstrap.min.css" />
  9 |     </head>
 10 |     <body>
 11 |         <script src="http://cdn.bootcss.com/jquery/1.11.2/jquery.min.js"></script>
 12 |         <script src="http://cdn.bootcss.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
 13 | 
 14 |         <div class="container-fluid">
 15 |             <h1>豆瓣租房搜索</h1>
 16 |             <div class="row" style="margin-left:3px;">
 17 |                 <div clss="col-md-12">
 18 |                     <form action="/" method="POST" class="form-inline">
 19 |                         <div class="form-group">
 20 |                             <label for="keyword">请输入关键词：</label>
 21 |                             <input type="text" class="form-control" id="keyword"
 22 |                                 name="keyword" value="{{ keyword }}" placeholder="请输入搜索关键字" />
 23 |                         </div>
 24 |                         <div class="form-group">
 25 |                             <label for="is_order_update_time">
 26 |                                 排序：
 27 |                             </label>
 28 |                             <select name="is_order_update_time" class="form-control">
 29 |                                 <option value="0" {% if is_order_update_time == 0 %} selected {% endif %}>发帖时间倒序</option>
 30 |                                 <option value="1" {% if is_order_update_time == 1 %} selected {% endif %}>更新时间倒序</option>
 31 |                             </select>
 32 |                         </div>
 33 |                         <button type="submit" class="btn btn-primary">搜索</button>
 34 |                     </form>
 35 |                 </div>
 36 |             </div>
 37 |             <div class="row" style="margin-left:3px;margin-right:3px;">
 38 |                 <div clss="col-md-12">
 39 |                     <div class="table-responsive" style="padding-top:20px;">
 40 |                         <table class="table table-bordered table-hover table-condensed table-striped">
 41 |                             <thead>
 42 |                                 <tr>
 43 |                                     <th width="50%" class="info">标题</th>
 44 |                                     <th width="20%" class="info">发表/更新时间</th>
 45 |                                 </tr>
 46 |                             </thead>
 47 |                             <tbody>
 48 |                                 {% for item in topics %}
 49 |                                 <tr>
 50 |                                     <td><a href="{{ item["url"]}} " target="_blank">{{ item["title"] }}</a></td>
 51 |                                     {% if item["create_time"] %}
 52 |                                     <td>{{ item["create_time"] }}</td>
 53 |                                     {% else %}
 54 |                                     <td>{{ item["last_reply_time"] }}</td>
 55 |                                     {% endif %}
 56 |                                 </tr>
 57 |                                 {% endfor %}
 58 |                             </tbody>
 59 |                         </table>
 60 |                     </div>
 61 |                 </div>
 62 |             </div>
 63 |             <div class="row">
 64 |                 <div class="col-md-2" style="margin-top:30px;">
 65 |                     共 <span style="color:red;">{{ count }}</span> 条记录
 66 |                     共 <span style="color:red;">{{ page.max_page }}</span> 页
 67 |                 </div>
 68 |                 <div class="col-md-10">
 69 |                     <nav>
 70 |                       <ul class="pagination">
 71 |                         {% if page.cpage == 1 %}
 72 |                            <li class="disabled">
 73 |                               <span aria-hidden="true">第一页</span>
 74 |                            </li>
 75 |                             <li class="disabled">
 76 |                                 <span aria-hidden="true">&laquo;</span>
 77 |                             </li>
 78 |                         {% else %}
 79 |                            <li>
 80 |                                 <a href="javascript:gotoPage(1)">
 81 |                                     <span aria-hidden="true">第一页</span>
 82 |                                 </a>
 83 |                            </li>
 84 |                             <li>
 85 |                                 <a href="javascript:gotoPage({{ page.cpage - 1 }})" aria-label="Previous">
 86 |                                     <span aria-hidden="true">&laquo;</span>
 87 |                                 </a>
 88 |                             </li>
 89 |                         {% endif %}
 90 | 
 91 |                         {% for p in range(page.begin_page, page.end_page + 1) %}
 92 |                             {% if page.cpage == p %}
 93 |                                 <li class="active"><span aria-hidden="true">{{ p }}</span></li>
 94 |                             {% else %}
 95 |                                 <li><a href="javascript:gotoPage({{p}})">{{ p }}</a></li>
 96 |                             {% endif %}
 97 |                         {% endfor %}
 98 | 
 99 |                         {% if page.cpage == page.max_page %}
100 |                              <li class="disabled">
101 |                                 <span aria-hidden="true">&raquo;</span>
102 |                             </li>
103 |                             <li class="disabled">
104 |                                 <span aria-hidden="true">最后一页</span>
105 |                             </li>
106 |                         {% else %}
107 |                             <li>
108 |                                 <a href="javascript:gotoPage({{ page.cpage + 1 }})" aria-label="Next">
109 |                                     <span aria-hidden="true">&raquo;</span>
110 |                                 </a>
111 |                             </li>
112 |                             <li>
113 |                                 <a href="javascript:gotoPage({{page.max_page}})">
114 |                                     <span aria-hidden="true">最后一页</span>
115 |                                 </a>
116 |                             </li>
117 |                         {% endif %}
118 |                       </ul>
119 |                     </nav>
120 |                 </div>
121 |             </div>
122 |         </div>
123 |         <script type="text/javascript">
124 |             function gotoPage(cpage){
125 |                 $myform = $("form");
126 |                 $myform.append("<input type='hidden' name='cpage' value=" + cpage+ ">")
127 |                 $myform.submit();
128 |             }
129 |         </script>
130 |     </body>
131 | </html>
132 | 


--------------------------------------------------------------------------------
/spider.py:
--------------------------------------------------------------------------------
  1 | # coding: utf8
  2 | 
  3 | """
  4 | 豆瓣小组爬虫
  5 | """
  6 | 
  7 | from gevent import monkey
  8 | monkey.patch_all()
  9 | 
 10 | import time
 11 | import random
 12 | import logging
 13 | 
 14 | import gevent
 15 | import requests
 16 | from gevent.pool import Pool
 17 | from gevent.queue import Queue
 18 | from lxml import etree
 19 | from dbmixin import DBMixin
 20 | 
 21 | from config import (
 22 |     GROUP_LIST, GROUP_SUFFIX, USER_AGENT,
 23 |     POOL_SIZE, RULES, MAX_PAGE, WATCH_INTERVAL, PROXY_INTERVAL
 24 | )
 25 | from utils import Timer, ProxyManager
 26 | 
 27 | 
 28 | def get_logger(name):
 29 |     """logger
 30 |     """
 31 |     default_logger = logging.getLogger(name)
 32 |     default_logger.setLevel(logging.DEBUG)
 33 |     stream = logging.StreamHandler()
 34 |     stream.setLevel(logging.DEBUG)
 35 |     formatter = logging.Formatter("[%(levelname)s] %(asctime)s - %(message)s")
 36 |     stream.setFormatter(formatter)
 37 |     default_logger.addHandler(stream)
 38 |     return default_logger
 39 | 
 40 | 
 41 | logger = get_logger("douban_spider")
 42 | 
 43 | 
 44 | class HTTPError(Exception):
 45 | 
 46 |     """ HTTP状态码不是200异常 """
 47 | 
 48 |     def __init__(self, status_code, url):
 49 |         self.status_code = status_code
 50 |         self.url = url
 51 | 
 52 |     def __str__(self):
 53 |         return "%s HTTP %s" % (self.url, self.status_code)
 54 | 
 55 | 
 56 | class URLFetchError(Exception):
 57 | 
 58 |     """ HTTP请求结果为空异常 """
 59 | 
 60 |     def __init__(self, url):
 61 |         self.url = url
 62 | 
 63 |     def __str__(self):
 64 |         return "%s fetch failed!" % self.self.url
 65 | 
 66 | 
 67 | class DoubanSpider(DBMixin):
 68 | 
 69 |     """" 豆瓣爬虫 """
 70 | 
 71 |     def __init__(self, proxy_manager=None):
 72 |         self.result_page = self.db.result_page
 73 |         self.result_topic = self.db.result_topic
 74 |         self.cache = self.db.cache_page
 75 | 
 76 |         self.group_list = GROUP_LIST
 77 |         self.rules = RULES
 78 |         self.interval = WATCH_INTERVAL
 79 | 
 80 |         self.pool = Pool(size=POOL_SIZE)
 81 |         self.page_queue = Queue()
 82 |         self.topic_queue = Queue()
 83 | 
 84 |         self.proxy_manager = proxy_manager
 85 | 
 86 |     def fetch(self, url, timeout=10, retury_num=10):
 87 |         """发起HTTP请求
 88 | 
 89 |         @url, str, URL
 90 |         @timeout, int, 超时时间
 91 |         @retury_num, int, 重试次数
 92 |         """
 93 |         kwargs = {
 94 |             "headers": {
 95 |                 "User-Agent": USER_AGENT,
 96 |                 "Referer": "http://www.douban.com/"
 97 |             },
 98 |         }
 99 |         kwargs["timeout"] = timeout
100 |         resp = None
101 |         for i in range(retury_num):
102 |             try:
103 |                 # 是否启动代理
104 |                 if self.proxy_manager is not None:
105 |                     kwargs["proxies"] = {
106 |                         "http": self.proxy_manager.get_proxy()}
107 |                 resp = requests.get(url, **kwargs)
108 |                 if resp.status_code != 200:
109 |                     raise HTTPError(resp.status_code, url)
110 |                 break
111 |             except Exception as exc:
112 |                 logger.warn("%s %d failed!\n%s", url, i, str(exc))
113 |                 time.sleep(2)
114 |                 continue
115 |         if resp is None:
116 |             raise URLFetchError(url)
117 |         return resp.content.decode("utf8")
118 | 
119 |     def extract(self, regx, body, multi=False):
120 |         """解析元素,xpath语法
121 | 
122 |         @regx, str, 解析表达式
123 |         @body, unicode or element, 网页源码或元素
124 |         @multi, bool, 是否取多个
125 |         """
126 |         if isinstance(body, unicode):
127 |             body = etree.HTML(body)
128 |         res = body.xpath(regx)
129 |         if multi:
130 |             return res
131 |         return res[0] if res else None
132 | 
133 |     def run(self):
134 |         """run
135 |         """
136 |         all_greenlet = []
137 |         # 定时爬取
138 |         for group_url in self.group_list:
139 |             # timer = Timer(random.randint(0, self.interval), self.interval)
140 |             timer = Timer(random.randint(0, 2), self.interval)
141 |             greenlet = gevent.spawn(
142 |                 timer.run, self._init_page_tasks, group_url)
143 |             all_greenlet.append(greenlet)
144 |         # 生产 & 消费
145 |         all_greenlet.append(gevent.spawn(self._page_loop))
146 |         all_greenlet.append(gevent.spawn(self._topic_loop))
147 |         # 重载代理,10分
148 |         proxy_timer = Timer(PROXY_INTERVAL, PROXY_INTERVAL)
149 |         all_greenlet.append(
150 |             gevent.spawn(proxy_timer.run(self.reload_proxies)))
151 |         gevent.joinall(all_greenlet)
152 | 
153 |     def reload_proxies(self):
154 |         """重新加载代理
155 |         """
156 |         self.proxy_manager.reload_proxies()
157 | 
158 |     def _init_page_tasks(self, group_url):
159 |         """初始化页面任务
160 | 
161 |         @group_url, str, 小组URL
162 |         """
163 |         for page in range(MAX_PAGE):
164 |             base_url = "%s%s" % (group_url, GROUP_SUFFIX)
165 |             url = base_url % (page * 25)
166 |             self.page_queue.put(url)
167 | 
168 |     def _page_loop(self):
169 |         """page loop
170 |         """
171 |         while 1:
172 |             page_url = self.page_queue.get(block=True)
173 |             gevent.sleep(1)
174 |             self.pool.spawn(self._crawl_page, page_url)
175 | 
176 |     def _topic_loop(self):
177 |         """topic loop
178 |         """
179 |         while 1:
180 |             topic_url = self.topic_queue.get(block=True)
181 |             self.pool.spawn(self._crawl_detail, topic_url)
182 | 
183 |     def _crawl_page(self, url):
184 |         """爬取帖子
185 | 
186 |         @url, str, 当前页面URL
187 |         """
188 |         logger.info("processing page: %s", url)
189 |         html = self.fetch(url)
190 |         topic_urls = self.extract(
191 |             self.rules["url_list"], html, multi=True)
192 |         # 找出新增的帖子URL
193 |         diff_urls = self._diff_urls(topic_urls)
194 |         if not diff_urls:
195 |             logger.info("%s no update ...", url)
196 |             return
197 |         logger.info("%s new add : %d", url, len(diff_urls))
198 |         topic_list = self.extract(
199 |             self.rules["topic_item"], html, multi=True)
200 |         # 获取每一页的信息
201 |         topics = self._get_page_info(topic_list)
202 |         # 过滤,找到新增的和之前的帖子
203 |         new_topics, old_topics = self._filter_topics(topics, diff_urls)
204 |         # 保存每页的信息
205 |         self.result_page.insert(new_topics)
206 |         # 更新老帖子的时间和回复数
207 |         self._update_old_topics(old_topics)
208 |         # 初始化帖子任务
209 |         self._init_topic_tasks(diff_urls)
210 |         # 更新缓存
211 |         self._update_cache(diff_urls)
212 | 
213 |     def _get_page_info(self, topic_list):
214 |         """获取每一页的帖子基本信息
215 | 
216 |         @topic_list, list, 当前页的帖子项
217 |         """
218 |         topics = []
219 |         # 第一行是标题头,舍掉
220 |         for topic_item in topic_list[1:]:
221 |             topic = {}
222 |             topic["title"] = self.extract(self.rules["title"], topic_item)
223 |             topic["author"] = self.extract(self.rules["author"], topic_item)
224 |             topic["reply"] = self.extract(self.rules["reply"], topic_item) or 0
225 |             topic["last_reply_time"] = self.extract(
226 |                 self.rules["last_reply_time"], topic_item)
227 |             topic["url"] = self.extract(self.rules["url"], topic_item)
228 |             now = time.time()
229 |             topic["got_time"] = now
230 |             topic["last_update_time"] = now
231 |             topics.append(topic)
232 |         return topics
233 | 
234 |     @staticmethod
235 |     def _filter_topics(topics, diff_urls):
236 |         """过滤帖子,找出新增的和老的帖子
237 | 
238 |         @topics, list, 当前页所有帖子信息
239 |         @diff_urls, list, 新增的帖子URL
240 |         """
241 |         new_topics, old_topics = [], []
242 |         for topic in topics:
243 |             if topic["url"] in diff_urls:
244 |                 new_topics.append(topic)
245 |             else:
246 |                 old_topics.append(topic)
247 |         return new_topics, old_topics
248 | 
249 |     def _diff_urls(self, topic_urls):
250 |         """过滤重复帖子URL
251 | 
252 |         @topic_urls, list, 当前页所有帖子URL
253 |         """
254 |         # 与缓存比较
255 |         cache_urls = []
256 |         cursor = self.cache.find()
257 |         for item in cursor:
258 |             cache_urls.extend(item["urls"])
259 |         # 找出新增的URL
260 |         diff_urls = list(set(topic_urls) - set(cache_urls))
261 |         return diff_urls
262 | 
263 |     def _update_old_topics(self, old_topics):
264 |         """更新老帖子的信息,标题,回应时间和回复数量
265 | 
266 |         @old_topics, list, 老帖子列表
267 |         """
268 |         for topic in old_topics:
269 |             new_info = {
270 |                 "title": topic["title"],
271 |                 "reply": topic["reply"],
272 |                 "last_reply_time": topic["last_reply_time"],
273 |                 "last_update_time": time.time()
274 |             }
275 |             self.result_page.update(
276 |                 {"url": topic["url"]},
277 |                 {"$set": new_info}
278 |             )
279 |             logger.info("%s updated ...", topic["url"])
280 | 
281 |     def _init_topic_tasks(self, topic_urls):
282 |         """初始化帖子任务
283 | 
284 |         @topic_urls, list, 当前页面帖子的URL
285 |         """
286 |         for url in topic_urls:
287 |             self.topic_queue.put(url)
288 | 
289 |     def _update_cache(self, diff_urls):
290 |         """更新缓存
291 | 
292 |         @diff_urls, list, 新增的帖子URL
293 |         """
294 |         self.cache.insert(
295 |             {"got_time": time.time(), "urls": diff_urls})
296 | 
297 |     def _crawl_detail(self, url):
298 |         """爬取每个帖子的详情
299 | 
300 |         @url, str, 每个帖子的URL
301 |         """
302 |         logger.info("processing topic: %s", url)
303 |         html = self.fetch(url)
304 |         # 获取每一页的信息
305 |         topic = self._get_detail_info(html, url)
306 |         if not topic:
307 |             self.topic_queue.put(url)
308 |             return
309 |         topic["url"] = url
310 |         topic["got_time"] = time.time()
311 |         # 不存在 & 保存帖子的信息
312 |         if self.result_topic.find_one({"url": url}):
313 |             return
314 |         self.result_topic.insert(topic)
315 | 
316 |     def _get_detail_info(self, html, url):
317 |         """获取帖子详情
318 | 
319 |         @html, str, 页面
320 |         """
321 |         if u"机器人" in html:
322 |             logger.warn("%s 403.html", url)
323 |             return None
324 |         topic = {}
325 |         title = self.extract(self.rules["detail_title_sm"], html) \
326 |             or self.extract(self.rules["detail_title_lg"], html)
327 |         if title is None:
328 |             return None
329 |         topic["title"] = title.strip()
330 |         topic["create_time"] = self.extract(
331 |             self.rules["create_time"], html)
332 |         topic["author"] = self.extract(
333 |             self.rules["detail_author"], html)
334 |         topic["content"] = '\n'.join(
335 |             self.extract(self.rules["content"], html, multi=True))
336 |         return topic
337 | 
338 | 
339 | def main():
340 |     """ main """
341 |     proxy_manager = ProxyManager("./proxy_list.txt", 30)
342 |     spider = DoubanSpider(proxy_manager)
343 |     spider.run()
344 | 
345 | 
346 | if __name__ == "__main__":
347 |     main()
348 | 


--------------------------------------------------------------------------------
/proxy_list.txt:
--------------------------------------------------------------------------------
  1 | 36.250.74.88:8102
  2 | 124.88.67.53:83
  3 | 124.88.67.53:80
  4 | 117.168.235.126:8123
  5 | 111.9.232.121:8123
  6 | 54.177.191.78:80
  7 | 58.251.132.181:8888
  8 | 82.219.5.161:80
  9 | 125.39.66.68:80
 10 | 117.166.109.99:8123
 11 | 117.9.130.131:8118
 12 | 104.154.40.81:3128
 13 | 185.28.193.95:8080
 14 | 222.45.196.53:8118
 15 | 61.152.116.137:80
 16 | 124.202.169.98:8118
 17 | 196.201.218.202:117
 18 | 37.59.101.107:8080
 19 | 196.201.218.204:3197
 20 | 211.141.130.186:8118
 21 | 117.177.243.79:80
 22 | 201.210.155.247:8080
 23 | 117.170.54.148:8123
 24 | 201.87.210.214:3128
 25 | 196.201.218.199:510
 26 | 196.201.218.202:635
 27 | 117.177.249.29:80
 28 | 117.135.250.69:80
 29 | 117.135.250.69:82
 30 | 183.207.228.51:81
 31 | 183.207.228.51:80
 32 | 183.207.228.51:82
 33 | 183.207.228.51:83
 34 | 183.207.228.51:84
 35 | 183.207.228.51:85
 36 | 183.207.228.7:9003
 37 | 220.248.108.113:80
 38 | 54.207.81.191:3333
 39 | 124.202.182.254:8118
 40 | 123.121.75.27:8118
 41 | 124.88.67.82:80
 42 | 183.222.152.191:8123
 43 | 106.38.188.165:80
 44 | 85.10.248.24:3128
 45 | 222.137.123.3:8118
 46 | 46.243.114.226:3128
 47 | 58.213.48.108:80
 48 | 218.59.144.120:81
 49 | 218.59.144.120:80
 50 | 122.228.92.103:80
 51 | 183.207.228.7:8039
 52 | 183.207.228.7:8033
 53 | 190.15.192.120:3128
 54 | 112.18.50.189:8123
 55 | 124.88.67.33:83
 56 | 124.88.67.33:81
 57 | 124.88.67.33:80
 58 | 105.237.233.175:8080
 59 | 88.150.136.180:3128
 60 | 36.250.74.87:8101
 61 | 54.207.37.91:3333
 62 | 196.201.218.199:684
 63 | 109.230.252.222:3128
 64 | 201.87.244.211:5555
 65 | 162.248.162.37:80
 66 | 123.234.8.36:8089
 67 | 123.115.138.10:8118
 68 | 113.105.111.153:81
 69 | 196.201.218.202:1936
 70 | 114.252.81.13:8118
 71 | 122.96.59.106:81
 72 | 122.96.59.106:82
 73 | 196.201.218.202:8080
 74 | 221.5.69.51:80
 75 | 211.77.5.41:3128
 76 | 124.88.67.13:80
 77 | 124.88.67.13:83
 78 | 124.88.67.13:81
 79 | 177.206.30.154:3128
 80 | 36.250.74.88:80
 81 | 111.161.126.98:80
 82 | 125.161.13.62:8080
 83 | 118.126.4.46:8080
 84 | 202.119.25.227:9999
 85 | 219.142.192.196:993
 86 | 219.142.192.196:197
 87 | 219.142.192.196:198
 88 | 218.104.144.248:8080
 89 | 122.96.59.106:80
 90 | 190.204.249.243:8080
 91 | 200.169.142.73:80
 92 | 202.159.6.146:80
 93 | 117.166.233.116:8123
 94 | 196.201.218.202:118
 95 | 111.13.136.58:80
 96 | 101.4.136.100:9999
 97 | 124.88.67.6:80
 98 | 124.88.67.6:81
 99 | 124.88.67.40:843
100 | 54.207.81.157:3333
101 | 219.142.192.196:3483
102 | 42.121.105.155:8888
103 | 223.84.144.183:8123
104 | 111.206.81.248:80
105 | 198.200.46.197:8000
106 | 196.201.218.204:982
107 | 206.17.20.75:3128
108 | 196.201.218.199:1583
109 | 183.203.208.174:8118
110 | 117.177.249.23:80
111 | 124.202.208.18:8001
112 | 196.201.218.202:1275
113 | 101.4.60.47:80
114 | 117.167.78.252:8123
115 | 123.103.8.88:3129
116 | 190.206.182.189:8080
117 | 204.145.84.49:8888
118 | 117.167.203.167:8123
119 | 218.204.154.167:8118
120 | 121.14.138.56:81
121 | 202.102.22.182:80
122 | 183.219.162.253:8123
123 | 117.135.251.74:80
124 | 163.177.79.4:8102
125 | 183.207.228.56:80
126 | 119.6.144.74:843
127 | 124.202.182.206:8118
128 | 117.177.249.24:80
129 | 124.88.67.8:80
130 | 124.88.67.8:81
131 | 122.96.59.106:843
132 | 183.207.228.6:8252
133 | 196.201.218.202:66
134 | 1.85.2.254:8000
135 | 183.207.228.7:9789
136 | 97.77.104.22:80
137 | 222.45.196.50:8118
138 | 219.142.192.196:5419
139 | 60.12.109.28:80
140 | 58.215.137.108:9080
141 | 221.10.102.199:82
142 | 221.10.102.199:83
143 | 196.201.218.202:4463
144 | 113.142.37.248:80
145 | 58.253.238.243:80
146 | 117.171.38.74:8123
147 | 54.76.226.32:80
148 | 88.150.136.180:3129
149 | 188.165.223.173:8106
150 | 111.249.163.164:3128
151 | 115.43.122.27:3128
152 | 190.204.70.208:8080
153 | 202.55.23.144:80
154 | 211.141.130.254:8118
155 | 200.109.65.75:3128
156 | 190.206.170.190:8080
157 | 188.165.223.173:8101
158 | 114.174.13.187:80
159 | 198.50.214.21:3128
160 | 58.215.137.117:9080
161 | 27.140.80.231:9999
162 | 54.207.82.42:3333
163 | 91.221.99.14:8080
164 | 114.244.215.89:8118
165 | 58.251.78.71:8088
166 | 122.228.92.73:80
167 | 211.141.130.105:8118
168 | 219.141.225.149:80
169 | 201.232.104.7:8080
170 | 124.202.181.6:8118
171 | 124.88.67.25:81
172 | 124.88.67.25:80
173 | 211.141.130.114:8118
174 | 186.93.231.245:8080
175 | 124.156.246.99:3128
176 | 124.88.67.7:80
177 | 120.206.158.47:8123
178 | 120.203.149.162:8118
179 | 88.150.136.179:3128
180 | 88.150.136.179:3129
181 | 183.207.228.7:9888
182 | 188.165.223.173:8102
183 | 111.206.50.177:80
184 | 223.197.56.102:80
185 | 188.165.223.173:8124
186 | 188.165.223.173:8108
187 | 58.215.137.98:9080
188 | 124.202.181.146:8001
189 | 117.167.184.90:8123
190 | 120.24.235.192:8888
191 | 219.142.192.196:2949
192 | 211.141.130.56:8118
193 | 124.88.67.24:843
194 | 183.61.240.126:80
195 | 116.228.55.217:8003
196 | 200.113.169.210:8800
197 | 58.254.132.81:8001
198 | 196.201.218.199:80
199 | 210.101.131.232:8080
200 | 200.195.22.33:3128
201 | 202.114.144.15:8088
202 | 103.25.166.107:9999
203 | 188.165.223.173:8126
204 | 58.215.137.118:9080
205 | 219.142.192.196:671
206 | 222.122.122.237:8081
207 | 196.201.218.203:255
208 | 50.200.101.172:3128
209 | 183.207.228.8:90
210 | 192.99.6.103:3128
211 | 124.88.67.10:843
212 | 54.207.81.81:3333
213 | 222.124.112.74:8080
214 | 66.4.18.180:8080
215 | 183.207.228.7:8111
216 | 183.207.228.7:8118
217 | 222.29.86.119:3128
218 | 193.63.43.34:8000
219 | 222.45.194.122:8118
220 | 58.215.137.101:9080
221 | 115.28.238.147:8080
222 | 219.217.227.93:3128
223 | 111.1.91.137:8118
224 | 120.192.249.74:80
225 | 119.90.127.2:80
226 | 117.171.236.26:8123
227 | 183.207.228.6:8161
228 | 36.250.74.87:8104
229 | 36.250.74.87:8102
230 | 183.207.228.41:80
231 | 183.207.228.43:80
232 | 183.207.228.6:8542
233 | 110.159.103.219:8080
234 | 202.29.97.2:3128
235 | 183.223.152.15:8123
236 | 124.88.67.36:83
237 | 175.43.20.95:80
238 | 196.201.218.199:1009
239 | 113.31.72.141:80
240 | 117.165.175.217:8123
241 | 58.215.137.120:9080
242 | 200.195.138.46:3128
243 | 54.244.94.51:8888
244 | 183.203.208.163:8118
245 | 202.96.155.251:8888
246 | 58.252.69.194:8888
247 | 221.10.102.203:843
248 | 200.20.168.135:80
249 | 54.251.189.206:3128
250 | 124.156.246.98:3128
251 | 120.203.148.6:8118
252 | 111.13.136.59:80
253 | 124.202.183.74:8118
254 | 54.207.75.153:3333
255 | 183.207.228.9:87
256 | 183.207.228.9:86
257 | 183.207.228.9:84
258 | 183.207.228.9:85
259 | 183.207.228.9:82
260 | 183.207.228.9:80
261 | 183.207.228.9:83
262 | 183.207.228.9:81
263 | 183.207.228.9:88
264 | 124.88.67.10:80
265 | 124.88.67.10:83
266 | 124.88.67.10:81
267 | 183.207.228.51:86
268 | 124.202.177.26:8118
269 | 117.21.192.9:80
270 | 181.30.27.19:80
271 | 183.216.127.19:8123
272 | 200.171.186.103:3128
273 | 123.103.8.249:3129
274 | 183.219.3.254:8123
275 | 187.62.205.74:3128
276 | 117.162.2.171:8123
277 | 117.165.189.124:8123
278 | 54.149.140.120:3128
279 | 183.220.244.176:8123
280 | 117.168.146.147:8123
281 | 183.207.228.7:8518
282 | 117.168.26.58:8123
283 | 101.4.136.66:84
284 | 117.167.82.210:8123
285 | 183.62.60.100:80
286 | 117.173.20.195:8123
287 | 187.104.43.109:8080
288 | 50.18.182.20:443
289 | 124.88.67.13:843
290 | 211.141.130.108:8118
291 | 183.136.135.153:8080
292 | 117.164.210.147:8123
293 | 112.45.102.185:8123
294 | 183.131.144.204:80
295 | 196.201.218.203:767
296 | 58.215.137.88:9080
297 | 117.164.16.29:8123
298 | 181.110.248.53:8080
299 | 190.79.2.117:8080
300 | 177.66.246.173:8080
301 | 58.215.137.96:9080
302 | 94.23.205.32:3128
303 | 113.107.57.76:3128
304 | 196.201.218.203:1237
305 | 117.162.172.238:8123
306 | 91.225.102.8:8080
307 | 183.207.228.6:8117
308 | 183.207.228.6:8112
309 | 200.58.74.49:3129
310 | 211.77.5.38:80
311 | 114.249.52.100:8118
312 | 116.77.128.68:8118
313 | 219.142.192.196:426
314 | 124.202.181.226:8118
315 | 219.142.192.196:282
316 | 114.215.108.155:9999
317 | 219.142.192.196:6609
318 | 203.81.162.211:3128
319 | 54.207.29.11:3333
320 | 89.31.150.71:8080
321 | 58.215.137.103:9080
322 | 210.22.63.67:8080
323 | 177.87.250.17:3128
324 | 163.177.79.4:8101
325 | 101.3.39.76:80
326 | 189.14.34.236:3128
327 | 118.229.239.17:8080
328 | 122.94.164.62:8118
329 | 198.11.172.223:3128
330 | 112.155.219.131:8080
331 | 115.29.184.17:82
332 | 75.151.84.36:8080
333 | 117.170.84.51:8123
334 | 183.207.228.7:8888
335 | 201.59.11.252:3128
336 | 218.89.170.78:8888
337 | 202.29.235.130:3129
338 | 196.201.218.204:80
339 | 112.19.127.119:8123
340 | 111.40.194.104:81
341 | 204.186.103.5:3128
342 | 186.225.35.69:8080
343 | 54.207.82.20:3333
344 | 124.88.67.30:81
345 | 177.130.53.216:3128
346 | 60.195.3.180:8118
347 | 122.136.46.151:3128
348 | 183.222.159.109:8123
349 | 183.222.153.148:8123
350 | 183.207.228.42:80
351 | 124.202.169.162:8118
352 | 196.201.218.202:455
353 | 163.177.79.5:80
354 | 119.97.164.48:8085
355 | 201.211.105.34:8080
356 | 123.57.68.233:80
357 | 186.46.185.186:8080
358 | 212.175.35.253:3128
359 | 05.101.138.140:8123
360 | 117.163.138.186:8123
361 | 119.6.144.74:83
362 | 119.6.144.74:80
363 | 119.6.144.74:82
364 | 196.201.218.202:1327
365 | 124.88.67.81:83
366 | 183.207.228.7:8202
367 | 124.88.67.81:80
368 | 124.88.67.81:81
369 | 85.105.20.114:8080
370 | 122.136.46.151:80
371 | 124.88.67.33:82
372 | 27.131.47.131:8080
373 | 36.250.74.87:80
374 | 194.135.220.18:8081
375 | 183.222.155.107:8123
376 | 124.88.67.83:80
377 | 183.207.228.58:80
378 | 115.28.137.244:80
379 | 59.58.162.141:888
380 | 200.222.105.27:3128
381 | 183.203.208.169:8118
382 | 121.199.42.198:3129
383 | 124.202.182.206:8001
384 | 50.57.131.89:3128
385 | 183.131.144.204:443
386 | 218.56.227.204:8118
387 | 112.95.244.57:80
388 | 123.58.129.48:80
389 | 117.148.44.19:8123
390 | 198.20.85.20:8118
391 | 115.28.62.175:80
392 | 221.12.173.130:3128
393 | 196.201.218.203:8080
394 | 122.224.90.20:80
395 | 119.6.144.74:81
396 | 183.217.96.73:8123
397 | 210.65.10.76:3128
398 | 219.142.192.196:595
399 | 219.142.192.196:228
400 | 218.30.128.190:8888
401 | 196.201.218.203:4472
402 | 183.219.149.250:8123
403 | 115.238.225.26:80
404 | 58.215.137.92:9080
405 | 124.88.67.20:80
406 | 124.88.67.20:83
407 | 124.88.67.54:80
408 | 124.88.67.54:83
409 | 201.14.153.247:80
410 | 181.30.27.18:80
411 | 58.215.137.119:9080
412 | 124.6.135.170:3128
413 | 190.207.11.67:8080
414 | 182.92.64.30:8080
415 | 89.191.175.14:3130
416 | 1.9.21.21:3128
417 | 94.23.23.60:80
418 | 124.91.135.153:8118
419 | 106.37.177.251:3128
420 | 182.140.140.36:80
421 | 117.166.68.202:8123
422 | 211.141.130.96:8118
423 | 123.103.8.87:3129
424 | 31.15.48.12:80
425 | 122.228.92.73:8080
426 | 221.10.102.203:81
427 | 221.10.102.203:83
428 | 221.10.102.203:80
429 | 222.124.219.215:3128
430 | 52.74.108.142:3128
431 | 124.88.67.34:82
432 | 125.39.66.69:80
433 | 196.201.218.199:32
434 | 117.171.157.17:8123
435 | 41.202.250.102:8080
436 | 125.24.77.36:80
437 | 91.240.68.193:443
438 | 109.60.249.107:3128
439 | 117.177.249.19:80
440 | 123.57.144.221:80
441 | 183.220.246.251:8123
442 | 200.112.211.110:8080
443 | 122.141.243.131:80
444 | 117.178.163.99:8123
445 | 117.78.5.31:3128
446 | 124.202.181.182:8001
447 | 193.63.43.64:8000
448 | 113.105.224.95:80
449 | 189.55.182.120:3128
450 | 181.88.177.145:8080
451 | 219.142.192.196:2898
452 | 211.141.130.112:8118
453 | 196.201.218.204:341
454 | 122.94.30.65:8118
455 | 219.142.192.196:264
456 | 196.201.218.203:428
457 | 112.244.151.157:8118
458 | 195.189.123.135:3128
459 | 183.220.240.157:8123
460 | 211.68.122.171:80
461 | 60.195.222.71:8001
462 | 88.150.136.181:3129
463 | 88.150.136.181:3128
464 | 137.116.158.58:80
465 | 124.88.67.32:83
466 | 124.88.67.32:80
467 | 202.108.50.75:81
468 | 202.108.50.75:82
469 | 202.108.50.75:80
470 | 183.207.228.7:9018
471 | 122.96.59.106:83
472 | 189.35.247.162:3128
473 | 190.75.236.187:8080
474 | 61.156.3.166:80
475 | 122.228.92.73:3128
476 | 196.201.218.204:1116
477 | 113.105.224.86:80
478 | 196.201.218.202:1014
479 | 200.167.148.10:3128
480 | 58.215.137.181:9080
481 | 218.240.156.82:80
482 | 202.55.23.168:80
483 | 219.142.54.90:80
484 | 185.54.4.198:8080
485 | 125.62.22.47:9999
486 | 117.135.252.84:9999
487 | 183.207.228.7:8281
488 | 117.165.105.144:8123
489 | 223.99.188.73:8090
490 | 202.119.25.72:9999
491 | 183.207.228.6:8032
492 | 219.223.190.90:3128
493 | 54.64.49.100:8080
494 | 183.222.156.214:8123
495 | 177.141.16.119:3128
496 | 128.199.248.14:8888
497 | 199.168.80.169:8082
498 | 177.7.4.132:3128
499 | 113.140.25.4:81
500 | 117.166.23.42:8123
501 | 117.168.177.95:8123
502 | 117.173.235.124:8123
503 | 111.161.126.101:80
504 | 123.138.184.228:80
505 | 219.142.192.196:182
506 | 183.217.36.61:8123
507 | 190.78.167.86:8080
508 | 54.207.81.139:3333
509 | 60.195.222.71:8118
510 | 200.24.220.94:80
511 | 183.207.228.6:8207
512 | 118.254.213.237:80
513 | 183.223.40.235:8123
514 | 222.45.85.210:8118
515 | 119.254.88.53:8080
516 | 58.214.5.229:80
517 | 183.207.228.57:80
518 | 124.88.67.9:83
519 | 124.88.67.9:81
520 | 124.88.67.9:80
521 | 121.231.131.237:8118
522 | 58.215.183.48:9080
523 | 209.66.192.149:3128
524 | 194.54.163.133:3130
525 | 122.225.117.26:80
526 | 54.64.140.6:8080
527 | 223.86.223.182:8123
528 | 219.136.252.120:80
529 | 211.77.5.41:80
530 | 58.252.72.179:3128
531 | 114.80.182.132:80
532 | 124.88.67.40:81
533 | 177.220.152.117:8081
534 | 77.37.130.244:8080
535 | 192.99.186.75:8080
536 | 117.170.230.182:8123
537 | 109.111.233.166:7777
538 | 124.88.67.83:81
539 | 54.207.52.18:3333
540 | 124.88.67.83:82
541 | 87.237.44.27:8080
542 | 196.201.218.204:853
543 | 124.88.67.83:83
544 | 183.207.228.7:8429
545 | 119.28.3.200:83
546 | 198.50.200.158:8089
547 | 120.131.128.214:80
548 | 122.225.106.40:80
549 | 219.142.192.196:7009
550 | 111.40.196.68:80
551 | 182.118.31.110:80
552 | 183.207.228.6:8076
553 | 82.200.217.66:8080
554 | 181.30.27.21:3128
555 | 183.207.228.7:8160
556 | 183.207.228.6:9309
557 | 187.76.82.149:3128
558 | 202.96.247.113:80
559 | 1.85.2.253:8000
560 | 196.201.218.204:1423
561 | 201.88.2.74:8080
562 | 36.250.74.88:8101
563 | 183.207.228.60:80
564 | 36.250.74.88:8104
565 | 182.254.129.68:80
566 | 120.85.132.234:80
567 | 180.130.137.95:9797
568 | 180.97.75.20:8080
569 | 187.17.217.138:3128
570 | 124.88.67.40:82
571 | 117.171.136.82:8123
572 | 183.207.228.7:8047
573 | 14.139.59.51:3128
574 | 131.229.80.59:8080
575 | 111.161.126.100:80
576 | 62.205.34.88:8000
577 | 121.42.29.118:80
578 | 123.125.104.240:80
579 | 219.142.192.196:808
580 | 219.142.192.196:803
581 | 124.202.182.182:8118
582 | 123.125.104.242:80
583 | 183.207.228.7:8265
584 | 177.66.247.155:8080
585 | 54.207.81.223:3333
586 | 37.49.137.243:80
587 | 58.215.183.47:9080
588 | 117.139.28.34:8123
589 | 58.96.177.117:3128
590 | 163.177.79.4:80
591 | 118.96.31.91:3128
592 | 117.79.64.29:80
593 | 110.75.120.170:80
594 | 119.147.86.212:9090
595 | 202.5.194.7:3128
596 | 221.176.14.72:80
597 | 58.253.238.242:80
598 | 109.196.127.35:8888
599 | 223.84.194.201:8123
600 | 116.236.216.116:8080
601 | 196.201.218.203:296
602 | 14.140.222.220:80
603 | 114.112.91.97:90
604 | 60.29.248.142:8080
605 | 201.208.162.142:3128
606 | 124.88.67.6:82
607 | 124.88.67.6:83
608 | 183.207.228.9:89
609 | 124.202.181.170:8118
610 | 115.85.70.60:1234
611 | 188.165.223.173:8118
612 | 54.207.81.250:3333
613 | 117.168.117.88:8123
614 | 183.203.208.168:8118
615 | 196.201.218.202:52
616 | 183.207.228.6:8518
617 | 196.201.218.204:37
618 | 94.46.229.126:3128
619 | 196.201.218.203:2958
620 | 190.199.186.249:8080
621 | 1.192.239.70:8118
622 | 42.51.132.239:8080
623 | 101.4.136.101:9999
624 | 183.203.22.76:80
625 | 124.88.67.20:81
626 | 118.26.183.173:8080
627 | 196.201.218.202:2969
628 | 88.150.136.178:3129
629 | 88.150.136.178:3128
630 | 92.242.126.86:3128
631 | 1.227.56.236:3128
632 | 104.197.8.4:3128
633 | 101.4.136.104:9999
634 | 210.101.131.227:8080
635 | 188.165.223.173:8115
636 | 183.203.208.172:8118
637 | 123.125.1.52:80
638 | 188.165.223.173:8113
639 | 163.177.79.5:8101
640 | 113.105.224.66:80
641 | 62.109.30.95:3128
642 | 118.192.93.173:80
643 | 124.88.67.54:81
644 | 124.88.67.24:81
645 | 124.88.67.24:80
646 | 58.215.183.46:9080
647 | 201.252.246.37:9090
648 | 196.201.218.202:34
649 | 202.100.166.132:80
650 | 124.88.67.84:80
651 | 111.13.136.59:843
652 | 218.90.174.167:3128
653 | 122.228.92.103:8080
654 | 196.201.218.204:213
655 | 190.154.51.178:8080
656 | 124.248.205.25:80
657 | 46.50.175.146:3128
658 | 198.236.242.223:80
659 | 183.207.228.8:85
660 | 183.207.228.8:87
661 | 183.207.228.8:84
662 | 183.207.228.8:81
663 | 183.207.228.8:83
664 | 183.207.228.8:80
665 | 119.147.91.21:80
666 | 183.60.126.65:9080
667 | 46.201.240.145:3128
668 | 183.203.208.162:8118
669 | 183.207.228.7:8100
670 | 183.207.228.7:8106
671 | 217.25.209.34:3128
672 | 183.207.228.7:8818
673 | 183.207.228.7:9989
674 | 190.64.148.74:8080
675 | 202.119.25.70:9999
676 | 121.31.5.188:8080
677 | 188.132.226.2:80
678 | 220.248.41.106:110
679 | 123.125.19.44:80
680 | 54.207.82.54:3333
681 | 124.202.181.198:8118
682 | 221.10.102.199:81
683 | 202.43.148.168:3128
684 | 119.188.94.145:80
685 | 124.202.180.186:8118
686 | 124.156.253.118:3128
687 | 122.225.106.35:80
688 | 104.154.58.149:3128
689 | 211.77.5.38:3128
690 | 54.88.246.102:80
691 | 196.201.218.199:8346
692 | 183.222.154.22:8123
693 | 202.101.96.154:8888
694 | 131.109.42.105:80
695 | 111.1.32.118:8080
696 | 133.242.19.155:80
697 | 124.88.67.34:80
698 | 183.223.34.3:8123
699 | 120.198.243.111:80
700 | 202.119.25.71:9999
701 | 115.44.98.239:8118
702 | 124.88.67.24:83
703 | 104.131.98.194:3128
704 | 124.202.181.102:8118
705 | 219.142.192.196:412
706 | 219.142.192.196:414
707 | 39.184.7.167:8123
708 | 58.135.68.102:80
709 | 1.202.150.116:8080
710 | 196.201.218.204:397
711 | 193.189.87.106:8080
712 | 219.142.192.196:411
713 | 200.203.146.69:3128
714 | 196.201.218.203:3304
715 | 58.252.167.103:80
716 | 58.215.183.51:9080
717 | 124.88.67.33:843
718 | 219.246.65.143:3128
719 | 211.71.20.251:3128
720 | 190.78.193.10:8080
721 | 181.30.27.21:80
722 | 183.207.228.9:90
723 | 183.60.44.136:88
724 | 110.5.100.218:3128
725 | 124.88.67.35:81
726 | 124.88.67.35:83
727 | 124.88.67.35:80
728 | 125.39.66.66:80
729 | 222.46.123.163:80
730 | 181.30.27.18:3128
731 | 118.168.198.179:80
732 | 219.142.192.196:3121
733 | 133.242.209.79:80
734 | 54.207.81.237:3333
735 | 89.107.114.160:3128
736 | 119.254.84.90:80
737 | 124.202.183.246:8118
738 | 117.171.233.62:8123
739 | 218.59.144.95:80
740 | 218.59.144.95:81
741 | 103.42.253.36:3129
742 | 192.52.166.175:3128
743 | 217.28.93.26:8080
744 | 210.200.13.232:80
745 | 120.131.129.227:80
746 | 202.107.233.85:8080
747 | 196.201.218.202:1427
748 | 211.141.130.252:8118
749 | 192.210.203.76:3128
750 | 196.201.218.199:190
751 | 113.106.93.42:3128
752 | 61.166.149.204:8081
753 | 186.233.116.90:80
754 | 180.247.48.124:3128
755 | 183.222.159.247:8123
756 | 200.129.147.2:3128
757 | 58.215.137.112:9080
758 | 58.215.183.35:9080
759 | 121.10.252.139:3128
760 | 183.220.105.22:8123
761 | 111.13.136.57:80
762 | 54.148.249.37:8888
763 | 122.225.106.36:80
764 | 111.13.136.57:843
765 | 117.139.28.171:8123
766 | 114.32.254.106:8080
767 | 101.4.60.43:80
768 | 124.202.183.246:8001
769 | 179.218.51.129:3128
770 | 185.53.168.95:8118
771 | 183.207.228.6:9877
772 | 118.69.226.241:8888
773 | 219.142.192.196:9415
774 | 122.228.92.103:3128
775 | 219.141.225.204:80
776 | 117.177.249.22:80
777 | 183.207.228.54:86
778 | 183.207.228.54:85
779 | 183.207.228.54:80
780 | 115.231.96.120:80
781 | 116.113.17.6:8090
782 | 101.4.136.3:9999
783 | 121.15.230.126:9797
784 | 219.142.192.196:2108
785 | 124.88.67.34:843
786 | 196.201.218.202:3719
787 | 119.80.160.52:80
788 | 111.11.184.51:9999
789 | 183.222.96.58:8123
790 | 124.202.176.102:8118
791 | 117.177.249.28:80
792 | 183.207.228.6:8310
793 | 182.118.23.7:8081
794 | 119.187.113.218:81
795 | 125.39.66.67:80
796 | 190.200.59.139:8080
797 | 128.199.99.56:3128
798 | 211.141.133.100:8118
799 | 104.155.45.90:3128
800 | 183.220.247.47:8123
801 | 117.166.57.11:8123
802 | 196.41.56.27:3128
803 | 101.4.136.66:80
804 | 101.4.136.66:86
805 | 101.4.136.66:81
806 | 101.4.136.66:82
807 | 101.4.136.66:83
808 | 111.1.3.36:8000
809 | 101.227.252.130:8081
810 | 190.37.82.135:8080
811 | 189.85.17.33:8080
812 | 183.207.228.7:8094
813 | 183.207.228.7:8099
814 | 196.201.218.202:2639
815 | 108.166.205.163:3128
816 | 223.99.188.74:8090
817 | 54.173.249.44:3128
818 | 121.201.18.74:3128
819 | 118.193.11.35:80
820 | 58.215.137.107:9080
821 | 87.245.168.14:3128
822 | 196.201.218.199:449
823 | 183.207.228.6:8083
824 | 183.207.228.6:8337
825 | 117.167.230.213:8123
826 | 121.42.211.168:80
827 | 101.4.136.66:85
828 | 210.101.131.231:8080
829 | 111.13.136.58:843
830 | 223.83.227.96:8123
831 | 207.244.89.22:8081
832 | 5.101.138.140:8123
833 | 120.131.128.211:80
834 | 124.202.181.146:8118
835 | 101.4.136.103:9999
836 | 124.202.176.102:8001
837 | 111.161.126.99:80
838 | 189.8.93.202:3128
839 | 196.201.218.199:808
840 | 122.94.179.243:8118
841 | 183.207.228.6:85
842 | 183.207.228.6:89
843 | 206.15.240.120:8080
844 | 113.107.57.76:80
845 | 120.85.132.234:110
846 | 31.170.237.202:3128
847 | 183.207.237.11:80
848 | 183.207.237.11:82
849 | 183.207.237.11:81
850 | 183.207.237.11:84
851 | 183.207.237.11:83
852 | 183.207.237.11:85
853 | 183.207.237.11:86
854 | 202.106.169.142:80
855 | 60.18.147.42:80
856 | 195.189.123.134:3128
857 | 196.201.218.199:1601
858 | 183.222.157.233:8123
859 | 219.142.192.196:1348
860 | 117.168.3.34:8123
861 | 183.207.228.6:9000
862 | 183.207.228.44:80
863 | 200.93.63.196:8080
864 | 124.81.237.114:3128
865 | 124.202.178.22:8001
866 | 223.210.55.45:80
867 | 117.170.93.206:8123
868 | 110.78.148.105:80
869 | 219.142.192.196:919
870 | 124.88.67.84:81
871 | 183.220.245.232:8123
872 | 124.88.67.84:83
873 | 183.207.228.6:8188
874 | 124.88.67.8:83
875 | 101.4.136.6:9999
876 | 58.215.137.87:9080
877 | 190.200.46.91:8080
878 | 183.207.228.7:8274
879 | 196.201.218.204:336
880 | 114.141.47.173:80
881 | 113.105.224.87:80
882 | 58.215.137.182:9080
883 | 201.210.198.154:8080
884 | 58.96.189.70:3128
885 | 119.254.102.253:80
886 | 146.148.120.126:3128
887 | 183.207.224.48:80
888 | 41.217.172.19:8080
889 | 182.92.240.197:8080
890 | 188.166.51.208:80
891 | 124.88.67.34:83
892 | 124.88.67.34:81
893 | 211.68.122.171:8080
894 | 60.207.228.235:80
895 | 58.215.137.100:9080
896 | 58.215.137.109:9080
897 | 202.195.161.28:8088
898 | 104.236.128.223:80
899 | 54.207.81.154:3333
900 | 101.226.249.237:80
901 | 37.239.46.74:80
902 | 121.42.42.249:80
903 | 124.202.208.18:8118
904 | 219.133.31.120:8888
905 | 120.203.149.110:8118
906 | 125.39.68.130:80
907 | 183.207.228.7:80
908 | 183.207.228.7:86
909 | 117.163.74.190:8123
910 | 187.58.9.92:8080
911 | 109.169.1.254:3128
912 | 177.91.235.18:3128
913 | 110.232.94.39:8087
914 | 180.248.32.56:3128
915 | 190.200.41.21:8080
916 | 117.171.240.37:8123
917 | 


--------------------------------------------------------------------------------