├── .gitignore
├── crawler
    ├── src
    │   ├── test
    │   │   ├── resources
    │   │   │   └── html
    │   │   │   │   ├── p66ip.html
    │   │   │   │   ├── ip3366.html
    │   │   │   │   ├── kxdaili.html
    │   │   │   │   ├── coobobo.html
    │   │   │   │   └── kuaidaili.html
    │   │   └── java
    │   │   │   └── com
    │   │   │       └── mrkid
    │   │   │           └── proxy
    │   │   │               └── crawler
    │   │   │                   └── vender
    │   │   │                       ├── cnproxy
    │   │   │                           ├── CnProxyTest.java
    │   │   │                           └── CnProxyXsltTest.java
    │   │   │                       ├── ip3366
    │   │   │                           ├── Ip3366Test.java
    │   │   │                           └── Ip3366XsltTest.java
    │   │   │                       ├── p66ip
    │   │   │                           ├── P66IPTest.java
    │   │   │                           └── P66IPXsltTest.java
    │   │   │                       ├── coobobo
    │   │   │                           ├── CooboboTest.java
    │   │   │                           └── CooboboXsltTest.java
    │   │   │                       ├── kxdaili
    │   │   │                           ├── KxdailiTest.java
    │   │   │                           └── KxDailiXsltTest.java
    │   │   │                       ├── kuaidaili
    │   │   │                           ├── KuaidailiTest.java
    │   │   │                           └── KuaidailiXsltTest.java
    │   │   │                       ├── p881free
    │   │   │                           └── P881FreeCrawlerTest.java
    │   │   │                       └── XsltTestUtils.java
    │   └── main
    │   │   ├── java
    │   │       └── com
    │   │       │   └── mrkid
    │   │       │       └── proxy
    │   │       │           └── crawler
    │   │       │               ├── ProxyFetcher.java
    │   │       │               ├── vender
    │   │       │                   ├── xici
    │   │       │                   │   └── XiciCrawler.java
    │   │       │                   ├── haoip
    │   │       │                   │   └── HaoIPCrawler.java
    │   │       │                   ├── ip3366
    │   │       │                   │   └── Ip3366ProxyFetcher.java
    │   │       │                   ├── cnproxy
    │   │       │                   │   └── CnProxyFetcher.java
    │   │       │                   ├── kxdaili
    │   │       │                   │   └── KxDailiProxyFetcher.java
    │   │       │                   ├── p881free
    │   │       │                   │   └── P881FreeCrawler.java
    │   │       │                   ├── kuaidaili
    │   │       │                   │   └── KuaiDaiLiProxyFetcher.java
    │   │       │                   ├── p66ip
    │   │       │                   │   └── P66IPProxyFetcher.java
    │   │       │                   ├── coobobo
    │   │       │                   │   └── CooboboProxyFetcher.java
    │   │       │                   └── goubanjia
    │   │       │                   │   └── GoubanjiaProxyFetcher.java
    │   │       │               ├── crawler4j
    │   │       │                   ├── Crawl4jProxyFetcher.java
    │   │       │                   └── ProxyWebCrawler.java
    │   │       │               ├── ProxyCrawlerConfiguration.java
    │   │       │               └── ProxyCrawlerApp.java
    │   │   └── resources
    │   │       ├── xsl
    │   │           ├── kxdaili.xsl
    │   │           ├── cn-proxy.xsl
    │   │           ├── ip3366.xsl
    │   │           ├── coobobo.xsl
    │   │           ├── kuaidaili.xsl
    │   │           └── p66ip.xsl
    │   │       └── logback.xml
    └── pom.xml
├── core
    ├── src
    │   └── main
    │   │   ├── resources
    │   │       ├── application.properties
    │   │       └── application-mysql.properties
    │   │   └── java
    │   │       └── com
    │   │           └── mrkid
    │   │               └── proxy
    │   │                   ├── repository
    │   │                       ├── LowQualityProxyRepository.java
    │   │                       └── ProxyRepository.java
    │   │                   ├── model
    │   │                       ├── ProxyKey.java
    │   │                       ├── Proxy.java
    │   │                       └── LowQualityProxy.java
    │   │                   ├── dto
    │   │                       ├── ProxyListDTO.java
    │   │                       ├── ProxyCheckResponse.java
    │   │                       ├── ProxyDTO.java
    │   │                       └── AnonymityType.java
    │   │                   └── service
    │   │                       └── ProxyService.java
    └── pom.xml
├── checker
    ├── src
    │   └── main
    │   │   ├── java
    │   │       └── com
    │   │       │   └── mrkid
    │   │       │       └── proxy
    │   │       │           └── checker
    │   │       │               ├── ProxyCheckResponseWriter.java
    │   │       │               ├── utils
    │   │       │                   └── AddressUtils.java
    │   │       │               ├── writer
    │   │       │                   ├── PlainFormatWriter.java
    │   │       │                   └── SquidFormatWriter.java
    │   │       │               ├── ProxyCheckerConfiguration.java
    │   │       │               ├── ProxyCheckerApp.java
    │   │       │               └── ProxyChecker.java
    │   │   └── resources
    │   │       └── logback.xml
    └── pom.xml
├── all-in-one.sh
├── Dockerfile
├── etl
    ├── pom.xml
    └── src
    │   └── main
    │       └── java
    │           └── com
    │               └── mrkid
    │                   └── proxy
    │                       └── etl
    │                           └── EtlApp.java
├── archive
    ├── pom.xml
    └── src
    │   └── main
    │       └── java
    │           └── com
    │               └── mrkid
    │                   └── proxy
    │                       └── archive
    │                           └── ArchiveApp.java
├── README.md
└── pom.xml


/.gitignore:
--------------------------------------------------------------------------------
 1 | target
 2 | tomcat
 3 | test_data
 4 | .idea
 5 | *.iml
 6 | crawl
 7 | data
 8 | db
 9 | *.log*
10 | 


--------------------------------------------------------------------------------
/crawler/src/test/resources/html/p66ip.html:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wuxudong/proxy-checker/HEAD/crawler/src/test/resources/html/p66ip.html


--------------------------------------------------------------------------------
/crawler/src/test/resources/html/ip3366.html:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wuxudong/proxy-checker/HEAD/crawler/src/test/resources/html/ip3366.html


--------------------------------------------------------------------------------
/core/src/main/resources/application.properties:
--------------------------------------------------------------------------------
1 | spring.datasource.url=jdbc:h2:./db/proxy;DB_CLOSE_ON_EXIT=FALSE
2 | spring.datasource.driverClassName=org.h2.Driver
3 | spring.datasource.username=sa
4 | spring.datasource.password=
5 | spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
6 | spring.jpa.hibernate.ddl-auto=update


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/ProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler;
 2 | 
 3 | import com.mrkid.proxy.dto.ProxyDTO;
 4 | 
 5 | import java.util.List;
 6 | 
 7 | /**
 8 |  * User: xudong
 9 |  * Date: 11/12/2016
10 |  * Time: 3:56 PM
11 |  */
12 | public interface ProxyFetcher {
13 |     List<ProxyDTO> crawl() throws Exception;
14 | }
15 | 


--------------------------------------------------------------------------------
/core/src/main/resources/application-mysql.properties:
--------------------------------------------------------------------------------
1 | spring.datasource.url=jdbc:mysql://127.0.0.1/proxy?useUnicode=true&characterEncoding=utf-8
2 | spring.datasource.driverClassName=com.mysql.jdbc.Driver
3 | spring.datasource.username=root
4 | spring.datasource.password=root
5 | spring.jpa.database-platform=org.hibernate.dialect.MySQL5InnoDBDialect
6 | spring.jpa.hibernate.ddl-auto=update
7 | 
8 | 
9 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/cnproxy/CnProxyTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.cnproxy;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 7:06 PM
 9 |  */
10 | public class CnProxyTest {
11 |     @Test
12 |     public void test() throws Exception {
13 |         System.out.println(new CnProxyFetcher().crawl());
14 |     }
15 | }
16 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/ip3366/Ip3366Test.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.ip3366;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 7:06 PM
 9 |  */
10 | public class Ip3366Test {
11 |     @Test
12 |     public void test() throws Exception {
13 |         System.out.println(new Ip3366ProxyFetcher().crawl());
14 |     }
15 | }
16 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/p66ip/P66IPTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.p66ip;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 1:24 PM
 9 |  */
10 | public class P66IPTest {
11 |     @Test
12 |     public void test() throws Exception {
13 |         System.out.println(new P66IPProxyFetcher(false).crawl());
14 |     }
15 | 
16 | }
17 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/coobobo/CooboboTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.coobobo;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 7:06 PM
 9 |  */
10 | public class CooboboTest {
11 |     @Test
12 |     public void test() throws Exception {
13 |         System.out.println(new CooboboProxyFetcher(false).crawl());
14 |     }
15 | }
16 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/kxdaili/KxdailiTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.kxdaili;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 1:24 PM
 9 |  */
10 | public class KxdailiTest {
11 |     @Test
12 |     public void test() throws Exception {
13 |         System.out.println(new KxDailiProxyFetcher().crawl());
14 |     }
15 | 
16 | }
17 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/kuaidaili/KuaidailiTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.kuaidaili;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 1:24 PM
 9 |  */
10 | public class KuaidailiTest {
11 |     @Test
12 |     public void test() throws Exception {
13 |         System.out.println(new KuaiDaiLiProxyFetcher().crawl());
14 |     }
15 | 
16 | }
17 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/p881free/P881FreeCrawlerTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.p881free;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 15/12/2016
 8 |  * Time: 8:17 PM
 9 |  */
10 | public class P881FreeCrawlerTest {
11 |     @Test
12 |     public void crawl() throws Exception {
13 |         System.out.println(new P881FreeCrawler().crawl());
14 | 
15 |     }
16 | 
17 | }


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/coobobo/CooboboXsltTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.coobobo;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.XsltTestUtils;
 4 | import org.junit.Test;
 5 | 
 6 | /**
 7 |  * User: xudong
 8 |  * Date: 15/12/2016
 9 |  * Time: 1:24 PM
10 |  */
11 | public class CooboboXsltTest {
12 |     @Test
13 |     public void test() throws Exception {
14 |         XsltTestUtils.transform("/html/coobobo.html", "/xsl/coobobo.xsl", "utf-8");
15 |     }
16 | }
17 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/ip3366/Ip3366XsltTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.ip3366;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.XsltTestUtils;
 4 | import org.junit.Test;
 5 | 
 6 | /**
 7 |  * User: xudong
 8 |  * Date: 15/12/2016
 9 |  * Time: 1:24 PM
10 |  */
11 | public class Ip3366XsltTest {
12 | 
13 |     @Test
14 |     public void test() throws Exception {
15 |         XsltTestUtils.transform("/html/ip3366.html", "/xsl/ip3366.xsl", "gb2312");
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/ProxyCheckResponseWriter.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.checker;
 2 | 
 3 | import com.mrkid.proxy.dto.ProxyCheckResponse;
 4 | import com.mrkid.proxy.model.Proxy;
 5 | 
 6 | import java.io.Closeable;
 7 | 
 8 | /**
 9 |  * User: xudong
10 |  * Date: 12/12/2016
11 |  * Time: 5:40 PM
12 |  */
13 | public interface ProxyCheckResponseWriter extends Closeable {
14 |     boolean shouldWrite(ProxyCheckResponse response);
15 | 
16 |     void write(ProxyCheckResponse response);
17 | }
18 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/cnproxy/CnProxyXsltTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.cnproxy;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.XsltTestUtils;
 4 | import org.junit.Test;
 5 | 
 6 | /**
 7 |  * User: xudong
 8 |  * Date: 15/12/2016
 9 |  * Time: 1:24 PM
10 |  */
11 | public class CnProxyXsltTest {
12 |     @Test
13 |     public void test() throws Exception {
14 |         XsltTestUtils.transform("/html/cn-proxy.html", "/xsl/cn-proxy.xsl", "utf-8");
15 |     }
16 | 
17 | }
18 | 


--------------------------------------------------------------------------------
/all-in-one.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | #this script should be run as root
 4 | 
 5 | java -jar crawler/target/crawler-1.0-SNAPSHOT.jar
 6 | 
 7 | java -jar checker/target/checker-1.0-SNAPSHOT.jar
 8 | 
 9 | #remove all cache_peer
10 | sed -i.bak '/cache_peer/d' /etc/squid3/squid.conf
11 | 
12 | count=$(cat data/proxy.squid | grep HIGH_ANONYMITY | wc -l)
13 | echo Updated $count proxies.
14 | 
15 | cat data/proxy.squid | grep HIGH_ANONYMITY | sed 's/#HIGH_ANONYMITY.*//' >> /etc/squid3/squid.conf
16 | 
17 | squid3 -k reconfigure
18 | 
19 | 
20 | 
21 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/repository/LowQualityProxyRepository.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.repository;
 2 | 
 3 | import com.mrkid.proxy.model.LowQualityProxy;
 4 | import com.mrkid.proxy.model.Proxy;
 5 | import com.mrkid.proxy.model.ProxyKey;
 6 | import org.springframework.data.domain.Pageable;
 7 | import org.springframework.data.jpa.repository.JpaRepository;
 8 | 
 9 | import java.util.List;
10 | 
11 | /**
12 |  * User: xudong
13 |  * Date: 21/10/2016
14 |  * Time: 11:43 AM
15 |  */
16 | public interface LowQualityProxyRepository extends JpaRepository<LowQualityProxy, ProxyKey> {
17 | }
18 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/model/ProxyKey.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.model;
 2 | 
 3 | import lombok.Data;
 4 | 
 5 | import javax.persistence.Embeddable;
 6 | import java.io.Serializable;
 7 | 
 8 | /**
 9 |  * User: xudong
10 |  * Date: 20/10/2016
11 |  * Time: 7:50 PM
12 |  */
13 | @Data
14 | @Embeddable
15 | public class ProxyKey implements Serializable{
16 |     private String type;
17 | 
18 |     private String host;
19 | 
20 |     private int port;
21 | 
22 |     public ProxyKey() {
23 |     }
24 | 
25 |     public ProxyKey(String type, String host, int port) {
26 |         this.type = type;
27 |         this.host = host;
28 |         this.port = port;
29 |     }
30 | }
31 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/dto/ProxyListDTO.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.dto;
 2 | 
 3 | import lombok.Data;
 4 | 
 5 | import javax.xml.bind.annotation.XmlAccessType;
 6 | import javax.xml.bind.annotation.XmlAccessorType;
 7 | import javax.xml.bind.annotation.XmlElement;
 8 | import javax.xml.bind.annotation.XmlRootElement;
 9 | import java.util.ArrayList;
10 | import java.util.List;
11 | 
12 | /**
13 |  * User: xudong
14 |  * Date: 31/10/2016
15 |  * Time: 6:16 PM
16 |  */
17 | @XmlRootElement(name = "proxies")
18 | @XmlAccessorType(XmlAccessType.FIELD)
19 | @Data
20 | public class ProxyListDTO {
21 |     @XmlElement(name = "proxy")
22 |     private List<ProxyDTO> proxies = new ArrayList<ProxyDTO>();
23 | 
24 | }
25 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/model/Proxy.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.model;
 2 | 
 3 | import com.mrkid.proxy.dto.AnonymityType;
 4 | import lombok.Data;
 5 | import lombok.EqualsAndHashCode;
 6 | 
 7 | import javax.persistence.*;
 8 | import java.util.Date;
 9 | 
10 | /**
11 |  * User: xudong
12 |  * Date: 31/10/2016
13 |  * Time: 6:16 PM
14 |  */
15 | @Data
16 | @Entity
17 | public class Proxy {
18 | 
19 |     @EmbeddedId
20 |     private ProxyKey key;
21 | 
22 |     private boolean valid;
23 | 
24 |     private int anonymityType = AnonymityType.UNKNOWN.getKey();
25 | 
26 |     private int recentFailTimes = 0;
27 | 
28 |     private String location;
29 |     private Date lastCheckSuccess;
30 |     private Date lastCheckFail;
31 | 
32 |     private String source = "";
33 | }
34 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/model/LowQualityProxy.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.model;
 2 | 
 3 | import com.mrkid.proxy.dto.AnonymityType;
 4 | import lombok.Data;
 5 | 
 6 | import javax.persistence.*;
 7 | import java.util.Date;
 8 | 
 9 | /**
10 |  * Archive low quality proxies
11 |  * User: xudong
12 |  * Date: 31/10/2016
13 |  * Time: 6:16 PM
14 |  */
15 | @Data
16 | @Entity
17 | public class LowQualityProxy {
18 |     @EmbeddedId
19 |     private ProxyKey key;
20 | 
21 |     private boolean valid;
22 | 
23 |     private int proxyType = AnonymityType.UNKNOWN.getKey();
24 | 
25 |     private int recentFailTimes = 0;
26 | 
27 |     private String location;
28 |     private Date lastCheckSuccess;
29 |     private Date lastCheckFail;
30 | 
31 |     private String source = "";
32 | }
33 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/xsl/kxdaili.xsl:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | 
 3 | <xsl:stylesheet
 4 |         xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 5 |         version="1.0">
 6 | 
 7 |     <xsl:output method="xml" indent="yes"/>
 8 | 
 9 |     <xsl:template match="/">
10 |         <proxies>
11 |             <xsl:for-each select="//table/tbody/tr">
12 |                 <proxy>
13 |                     <host>
14 |                         <xsl:value-of select="td[1]"/>
15 |                     </host>
16 |                     <port>
17 |                         <xsl:value-of select="td[2]"/>
18 |                     </port>
19 |                 </proxy>
20 |             </xsl:for-each>
21 |         </proxies>
22 |     </xsl:template>
23 | </xsl:stylesheet>
24 | 
25 | 
26 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/dto/ProxyCheckResponse.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.dto;
 2 | 
 3 | import lombok.Data;
 4 | 
 5 | /**
 6 |  * User: xudong
 7 |  * Date: 31/10/2016
 8 |  * Time: 3:02 PM
 9 |  */
10 | @Data
11 | public class ProxyCheckResponse {
12 |     private String originIp;
13 |     private String remoteIp;
14 |     private ProxyDTO proxy;
15 | 
16 |     private boolean valid;
17 | 
18 |     private AnonymityType anonymityType = AnonymityType.UNKNOWN;
19 | 
20 |     public ProxyCheckResponse() {
21 |     }
22 | 
23 | 
24 |     public ProxyCheckResponse(String originIp, String remoteIp, ProxyDTO proxy, boolean valid) {
25 |         this.originIp = originIp;
26 |         this.remoteIp = remoteIp;
27 |         this.proxy = proxy;
28 |         this.valid = valid;
29 |     }
30 | 
31 | }
32 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/repository/ProxyRepository.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.repository;
 2 | 
 3 | import com.mrkid.proxy.model.Proxy;
 4 | import com.mrkid.proxy.model.ProxyKey;
 5 | import org.springframework.data.domain.Pageable;
 6 | import org.springframework.data.jpa.repository.JpaRepository;
 7 | 
 8 | import java.util.List;
 9 | 
10 | /**
11 |  * User: xudong
12 |  * Date: 21/10/2016
13 |  * Time: 11:43 AM
14 |  */
15 | public interface ProxyRepository extends JpaRepository<Proxy, ProxyKey> {
16 | 
17 |     List<Proxy> findByValidIsTrue();
18 | 
19 |     List<Proxy> findByValidIsTrueOrRecentFailTimesLessThanOrderByKeyHost(int failTimeLimit, Pageable page);
20 | 
21 |     List<Proxy> findByValidIsFalseAndRecentFailTimesGreaterThanEqualOrderByKeyHost(int failTimeLimit, Pageable page);
22 | }
23 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/dto/ProxyDTO.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.dto;
 2 | 
 3 | import lombok.Data;
 4 | import lombok.EqualsAndHashCode;
 5 | import org.apache.commons.lang3.StringUtils;
 6 | 
 7 | import java.util.Date;
 8 | 
 9 | /**
10 |  * User: xudong
11 |  * Date: 31/10/2016
12 |  * Time: 6:16 PM
13 |  */
14 | @Data
15 | @EqualsAndHashCode(of = {"type", "host"})
16 | public class ProxyDTO {
17 |     private String type = "http";
18 |     private String host;
19 |     private int port;
20 | 
21 |     private String location;
22 |     private Date lastCheckSuccess;
23 |     private Date lastCheckFail;
24 | 
25 |     private String source = "";
26 | 
27 |     public void setLocation(String location) {
28 |         this.location = StringUtils.isNotBlank(location) ? location.trim() : "";
29 |     }
30 | }
31 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/utils/AddressUtils.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.checker.utils;
 2 | 
 3 | import org.jsoup.Jsoup;
 4 | 
 5 | import java.io.IOException;
 6 | import java.util.regex.Matcher;
 7 | import java.util.regex.Pattern;
 8 | 
 9 | /**
10 |  * User: xudong
11 |  * Date: 04/11/2016
12 |  * Time: 1:13 PM
13 |  */
14 | public class AddressUtils {
15 |     public static String getMyPublicIp() throws IOException {
16 |         final String body = Jsoup.connect("http://1212.ip138.com/ic.asp").get().body().text();
17 |         final Matcher matcher = Pattern.compile(".*\\[(.*)\\].*").matcher(body);
18 | 
19 |         if (matcher.find()) {
20 |             return matcher.group(1);
21 |         } else {
22 |             throw new RuntimeException("can not parse ip from " + body);
23 |         }
24 |     }
25 | }
26 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/xsl/cn-proxy.xsl:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | 
 3 | <xsl:stylesheet
 4 |         xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 5 |         version="1.0">
 6 | 
 7 |     <xsl:output method="xml" indent="yes"/>
 8 | 
 9 |     <xsl:template match="/">
10 |         <proxies>
11 |             <xsl:for-each select="//table/tbody/tr">
12 |                 <proxy>
13 |                     <ip>
14 |                         <xsl:value-of select="td[1]"/>
15 |                     </ip>
16 |                     <port>
17 |                         <xsl:value-of select="td[2]"/>
18 |                     </port>
19 |                     <location>
20 |                         <xsl:value-of select="td[3]"/>
21 |                     </location>
22 |                 </proxy>
23 |             </xsl:for-each>
24 |         </proxies>
25 |     </xsl:template>
26 | </xsl:stylesheet>
27 | 
28 | 
29 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/xsl/ip3366.xsl:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | 
 3 | <xsl:stylesheet
 4 |         xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 5 |         version="1.0">
 6 | 
 7 |     <xsl:output method="xml" indent="yes"/>
 8 | 
 9 |     <xsl:template match="/">
10 |         <proxies>
11 |             <xsl:for-each select="//table/tbody/tr">
12 |                 <proxy>
13 |                     <host>
14 |                         <xsl:value-of select="td[1]"/>
15 |                     </host>
16 |                     <port>
17 |                         <xsl:value-of select="td[2]"/>
18 |                     </port>
19 |                     <location>
20 |                         <xsl:value-of select="td[5]"/>
21 |                     </location>
22 |                 </proxy>
23 |             </xsl:for-each>
24 |         </proxies>
25 |     </xsl:template>
26 | </xsl:stylesheet>
27 | 
28 | 
29 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/xsl/coobobo.xsl:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | 
 3 | <xsl:stylesheet
 4 |         xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 5 |         version="1.0">
 6 | 
 7 |     <xsl:output method="xml" indent="yes"/>
 8 | 
 9 |     <xsl:template match="/">
10 |         <proxies>
11 |             <xsl:for-each select="//table/tbody/tr">
12 |                 <proxy>
13 |                     <host>
14 |                         <xsl:value-of select="td[1]"/>
15 |                     </host>
16 |                     <port>
17 |                         <xsl:value-of select="td[2]"/>
18 |                     </port>
19 |                     <location>
20 |                         <xsl:value-of select="td[4]"/>
21 |                     </location>
22 |                 </proxy>
23 |             </xsl:for-each>
24 |         </proxies>
25 |     </xsl:template>
26 | </xsl:stylesheet>
27 | 
28 | 
29 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/xsl/kuaidaili.xsl:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | 
 3 | <xsl:stylesheet
 4 |         xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 5 |         version="1.0">
 6 | 
 7 |     <xsl:output method="xml" indent="yes"/>
 8 | 
 9 |     <xsl:template match="/">
10 |         <proxies>
11 |             <xsl:for-each select="//table/tbody/tr">
12 |                 <proxy>
13 |                     <host>
14 |                         <xsl:value-of select="td[1]"/>
15 |                     </host>
16 |                     <port>
17 |                         <xsl:value-of select="td[2]"/>
18 |                     </port>
19 |                     <location>
20 |                         <xsl:value-of select="td[6]"/>
21 |                     </location>
22 |                 </proxy>
23 |             </xsl:for-each>
24 |         </proxies>
25 |     </xsl:template>
26 | </xsl:stylesheet>
27 | 
28 | 
29 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/p66ip/P66IPXsltTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.p66ip;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.XsltTestUtils;
 4 | import org.apache.tika.io.IOUtils;
 5 | import org.jsoup.Jsoup;
 6 | import org.jsoup.helper.W3CDom;
 7 | import org.junit.Test;
 8 | import org.w3c.dom.Document;
 9 | 
10 | import javax.xml.transform.Transformer;
11 | import javax.xml.transform.TransformerFactory;
12 | import javax.xml.transform.dom.DOMSource;
13 | import javax.xml.transform.stream.StreamResult;
14 | import javax.xml.transform.stream.StreamSource;
15 | import java.io.InputStream;
16 | 
17 | /**
18 |  * User: xudong
19 |  * Date: 15/12/2016
20 |  * Time: 1:24 PM
21 |  */
22 | public class P66IPXsltTest {
23 | 
24 |     @Test
25 |     public void test() throws Exception {
26 |         XsltTestUtils.transform("/html/p66ip.html", "/xsl/p66ip.xsl", "gb2312");
27 |     }
28 | 
29 | }
30 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/kxdaili/KxDailiXsltTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.kxdaili;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.XsltTestUtils;
 4 | import org.apache.tika.io.IOUtils;
 5 | import org.jsoup.Jsoup;
 6 | import org.jsoup.helper.W3CDom;
 7 | import org.junit.Test;
 8 | import org.w3c.dom.Document;
 9 | 
10 | import javax.xml.transform.Transformer;
11 | import javax.xml.transform.TransformerFactory;
12 | import javax.xml.transform.dom.DOMSource;
13 | import javax.xml.transform.stream.StreamResult;
14 | import javax.xml.transform.stream.StreamSource;
15 | import java.io.InputStream;
16 | 
17 | /**
18 |  * User: xudong
19 |  * Date: 15/12/2016
20 |  * Time: 1:24 PM
21 |  */
22 | public class KxDailiXsltTest {
23 | 
24 |     @Test
25 |     public void test() throws Exception {
26 |         XsltTestUtils.transform("/html/kxdaili.html", "/xsl/kxdaili.xsl", "utf-8");
27 |     }
28 | }
29 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/kuaidaili/KuaidailiXsltTest.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.kuaidaili;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.XsltTestUtils;
 4 | import org.apache.tika.io.IOUtils;
 5 | import org.jsoup.Jsoup;
 6 | import org.jsoup.helper.W3CDom;
 7 | import org.junit.Test;
 8 | import org.w3c.dom.Document;
 9 | 
10 | import javax.xml.transform.Transformer;
11 | import javax.xml.transform.TransformerFactory;
12 | import javax.xml.transform.dom.DOMSource;
13 | import javax.xml.transform.stream.StreamResult;
14 | import javax.xml.transform.stream.StreamSource;
15 | import java.io.InputStream;
16 | 
17 | /**
18 |  * User: xudong
19 |  * Date: 15/12/2016
20 |  * Time: 1:24 PM
21 |  */
22 | public class KuaidailiXsltTest {
23 | 
24 |     @Test
25 |     public void test() throws Exception {
26 |         XsltTestUtils.transform("/html/kuaidaili.html", "/xsl/kuaidaili.xsl", "utf-8");
27 |     }
28 | 
29 | }
30 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/dto/AnonymityType.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.dto;
 2 | 
 3 | import com.fasterxml.jackson.annotation.JsonCreator;
 4 | import com.fasterxml.jackson.annotation.JsonValue;
 5 | 
 6 | import java.util.Arrays;
 7 | 
 8 | /**
 9 |  * User: xudong
10 |  * Date: 02/12/2016
11 |  * Time: 8:45 AM
12 |  */
13 | 
14 | 
15 | public enum AnonymityType {
16 |     UNKNOWN(0),
17 |     TRANSPARENT_PROXY(1),
18 |     ANONYMOUS_PROXY(2),
19 |     DISTORTING_PROXY(3),
20 |     HIGH_ANONYMITY_PROXY(4);
21 | 
22 |     private int key;
23 | 
24 |     AnonymityType(int key) {
25 |         this.key = key;
26 |     }
27 | 
28 | 
29 |     @JsonValue
30 |     public int getKey() {
31 |         return key;
32 |     }
33 | 
34 |     @JsonCreator
35 |     public static AnonymityType create(int i) {
36 |         return Arrays.stream(AnonymityType.values()).filter(t -> t.key == i).findFirst().orElseGet(() ->
37 |                 UNKNOWN);
38 |     }
39 | }
40 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/xsl/p66ip.xsl:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | 
 3 | <xsl:stylesheet
 4 |         version="2.0"
 5 |         xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
 6 |         xmlns:xhtml="http://www.w3.org/1999/xhtml">
 7 | 
 8 |     <xsl:output method="xml" indent="yes"/>
 9 | 
10 |     <xsl:template match="/">
11 |         <proxies>
12 |             <xsl:for-each select="(//xhtml:table)[3]/xhtml:tbody/xhtml:tr[position()>1]">
13 |                 <proxy>
14 |                     <host>
15 |                         <xsl:value-of select="xhtml:td[1]"/>
16 |                     </host>
17 |                     <port>
18 |                         <xsl:value-of select="xhtml:td[2]"/>
19 |                     </port>
20 |                     <location>
21 |                         <xsl:value-of select="xhtml:td[3]"/>
22 |                     </location>
23 |                 </proxy>
24 |             </xsl:for-each>
25 |         </proxies>
26 | 
27 |     </xsl:template>
28 | </xsl:stylesheet>
29 | 
30 | 
31 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM minimum2scp/squid
 2 | 
 3 | # From https://github.com/dockerfile/java/blob/master/oracle-java8/Dockerfile
 4 | RUN \
 5 |   sudo apt-get install software-properties-common -y && \
 6 |   echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
 7 |   add-apt-repository -y ppa:webupd8team/java && \
 8 |   apt-get update && \
 9 |   apt-get install -y --allow-unauthenticated oracle-java8-installer && \
10 |   rm -rf /var/lib/apt/lists/* && \
11 |   rm -rf /var/cache/oracle-jdk8-installer
12 | 
13 | # Define commonly used JAVA_HOME variable
14 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
15 | 
16 | # Wifiyou
17 | RUN \
18 |   ln -s /etc/squid /etc/squid3 && \
19 |   mkdir -p /opt/wifiyou/proxy-checker/crawler/target/ && \
20 |   mkdir -p /opt/wifiyou/proxy-checker/checker/target/
21 | 
22 | # Define working directory.
23 | WORKDIR /opt/wifiyou/proxy-checker/
24 | 
25 | COPY crawler/target/crawler-1.0-SNAPSHOT.jar crawler/target/
26 | COPY checker/target/checker-1.0-SNAPSHOT.jar checker/target/
27 | COPY all-in-one.sh update
28 | 
29 | RUN chmod +x /opt/wifiyou/proxy-checker/update
30 | 


--------------------------------------------------------------------------------
/checker/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <configuration>
 3 |     <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
 4 |         <layout class="ch.qos.logback.classic.PatternLayout">
 5 |             <Pattern>
 6 |                 %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
 7 |             </Pattern>
 8 |         </layout>
 9 |     </appender>
10 | 
11 |     <appender name="rollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
12 |         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
13 |             <level>INFO</level>
14 |         </filter>
15 |         <file>checker.log</file>
16 |         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
17 |             <fileNamePattern>checker.log.%d{yyyy-MM-dd}</fileNamePattern>
18 |             <maxHistory>30</maxHistory>
19 |         </rollingPolicy>
20 | 
21 |         <encoder>
22 |             <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{35} - %msg%n
23 |             </pattern>
24 |         </encoder>
25 | 
26 |     </appender>
27 | 
28 |     <root level="INFO">
29 |         <appender-ref ref="rollingFile"/>
30 |     </root>
31 | </configuration>
32 | 


--------------------------------------------------------------------------------
/crawler/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <configuration>
 3 |     <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
 4 |         <layout class="ch.qos.logback.classic.PatternLayout">
 5 |             <Pattern>
 6 |                 %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
 7 |             </Pattern>
 8 |         </layout>
 9 |     </appender>
10 | 
11 |     <appender name="rollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
12 |         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
13 |             <level>INFO</level>
14 |         </filter>
15 |         <file>crawler.log</file>
16 |         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
17 |             <fileNamePattern>crawler.log.%d{yyyy-MM-dd}</fileNamePattern>
18 |             <maxHistory>30</maxHistory>
19 |         </rollingPolicy>
20 | 
21 |         <encoder>
22 |             <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{35} - %msg%n
23 |             </pattern>
24 |         </encoder>
25 | 
26 |     </appender>
27 | 
28 |     <root level="INFO">
29 |         <appender-ref ref="rollingFile"/>
30 |     </root>
31 | </configuration>
32 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/writer/PlainFormatWriter.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.checker.writer;
 2 | 
 3 | import com.mrkid.proxy.checker.ProxyCheckResponseWriter;
 4 | import com.mrkid.proxy.dto.ProxyCheckResponse;
 5 | 
 6 | import java.io.File;
 7 | import java.io.FileWriter;
 8 | import java.io.IOException;
 9 | import java.io.PrintWriter;
10 | 
11 | /**
12 |  * User: xudong
13 |  * Date: 16/12/2016
14 |  * Time: 5:25 PM
15 |  */
16 | public class PlainFormatWriter implements ProxyCheckResponseWriter {
17 |     final PrintWriter writer;
18 | 
19 |     public PlainFormatWriter(File dataDirectory) throws IOException {
20 |         writer = new PrintWriter(new FileWriter(new File(dataDirectory,
21 |                 "proxy.plain")));
22 |     }
23 | 
24 |     @Override
25 |     public boolean shouldWrite(ProxyCheckResponse response) {
26 |         return response.isValid() && "http".equalsIgnoreCase(response.getProxy().getType());
27 |     }
28 | 
29 |     @Override
30 |     public void write(ProxyCheckResponse response) {
31 |         writer.println(response.getProxy().getHost() + ":" + response.getProxy().getPort());
32 | 
33 |     }
34 | 
35 |     @Override
36 |     public void close() throws IOException {
37 |         writer.close();
38 |     }
39 | }
40 | 


--------------------------------------------------------------------------------
/etl/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <parent>
 8 |         <groupId>com.mrkid.proxy</groupId>
 9 |         <artifactId>parent</artifactId>
10 |         <version>1.0-SNAPSHOT</version>
11 |     </parent>
12 | 
13 |     <groupId>com.mrkid.proxy</groupId>
14 |     <artifactId>etl</artifactId>
15 | 
16 |     <dependencies>
17 |         <dependency>
18 |             <groupId>org.springframework.boot</groupId>
19 |             <artifactId>spring-boot-starter</artifactId>
20 |         </dependency>
21 | 
22 |         <dependency>
23 |             <groupId>com.mrkid.proxy</groupId>
24 |             <artifactId>core</artifactId>
25 |         </dependency>
26 | 
27 |     </dependencies>
28 | 
29 |     <build>
30 |         <plugins>
31 |             <plugin>
32 |                 <groupId>org.springframework.boot</groupId>
33 |                 <artifactId>spring-boot-maven-plugin</artifactId>
34 |             </plugin>
35 | 
36 |         </plugins>
37 |     </build>
38 | 
39 | 
40 | 
41 | </project>


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/xici/XiciCrawler.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.xici;
 2 | 
 3 | import com.mrkid.proxy.crawler.ProxyFetcher;
 4 | import com.mrkid.proxy.dto.ProxyDTO;
 5 | import org.jsoup.Jsoup;
 6 | import org.jsoup.nodes.Document;
 7 | import org.jsoup.nodes.Element;
 8 | 
 9 | import java.util.Arrays;
10 | import java.util.List;
11 | import java.util.stream.Collectors;
12 | 
13 | /**
14 |  * User: xudong
15 |  * Date: 03/11/2016
16 |  * Time: 12:47 PM
17 |  */
18 | public class XiciCrawler implements ProxyFetcher {
19 | 
20 |     @Override
21 |     public List<ProxyDTO> crawl() throws Exception {
22 |         Document doc = Jsoup.connect("http://api.xicidaili.com/free2016.txt").timeout(30000).get();
23 | 
24 |         final Element body = doc.body();
25 | 
26 |         final List<ProxyDTO> proxies = Arrays.stream(body.text().split("\\s")).map(s -> {
27 |             final String[] token = s.split(":");
28 |             ProxyDTO proxy = new ProxyDTO();
29 |             proxy.setType("http");
30 |             proxy.setHost(token[0]);
31 |             proxy.setPort(Integer.valueOf(token[1]));
32 |             proxy.setSource("XICI");
33 | 
34 |             return proxy;
35 | 
36 |         }).collect(Collectors.toList());
37 | 
38 |         return proxies;
39 | 
40 |     }
41 | 
42 | }
43 | 


--------------------------------------------------------------------------------
/crawler/src/test/java/com/mrkid/proxy/crawler/vender/XsltTestUtils.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender;
 2 | 
 3 | import org.apache.tika.io.IOUtils;
 4 | import org.jsoup.Jsoup;
 5 | import org.jsoup.helper.W3CDom;
 6 | import org.junit.Test;
 7 | import org.w3c.dom.Document;
 8 | 
 9 | import javax.xml.transform.Transformer;
10 | import javax.xml.transform.TransformerException;
11 | import javax.xml.transform.TransformerFactory;
12 | import javax.xml.transform.dom.DOMSource;
13 | import javax.xml.transform.stream.StreamResult;
14 | import javax.xml.transform.stream.StreamSource;
15 | import java.io.IOException;
16 | import java.io.InputStream;
17 | 
18 | /**
19 |  * User: xudong
20 |  * Date: 16/12/2016
21 |  * Time: 3:05 PM
22 |  */
23 | public class XsltTestUtils {
24 | 
25 |      public static void transform(String htmlPath, String xslPath, String htmEncoding) throws IOException,
26 |              TransformerException {
27 |         try (InputStream htmlStream = XsltTestUtils.class.getResourceAsStream(htmlPath);
28 |              InputStream xslStream = XsltTestUtils.class.getResourceAsStream(xslPath);
29 |         ) {
30 |             Document document = new W3CDom().fromJsoup(Jsoup.parse(IOUtils.toString(htmlStream, htmEncoding)));
31 | 
32 |             Transformer transformer = TransformerFactory.newInstance().newTransformer(new StreamSource(xslStream));
33 | 
34 |             transformer.transform(new DOMSource(document), new StreamResult(System.out));
35 |         }
36 |     }
37 | }
38 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/haoip/HaoIPCrawler.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.haoip;
 2 | 
 3 | import com.mrkid.proxy.crawler.ProxyFetcher;
 4 | import com.mrkid.proxy.dto.ProxyDTO;
 5 | import org.jsoup.Jsoup;
 6 | import org.jsoup.nodes.Document;
 7 | import org.jsoup.nodes.Element;
 8 | import org.jsoup.select.Elements;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | import java.util.stream.Collectors;
13 | 
14 | /**
15 |  * User: xudong
16 |  * Date: 03/11/2016
17 |  * Time: 12:47 PM
18 |  */
19 | public class HaoIPCrawler implements ProxyFetcher {
20 | 
21 |     @Override
22 |     public List<ProxyDTO> crawl() throws Exception {
23 |         Document doc = Jsoup.connect("http://haoip.cc/tiqu.htm").timeout(30000).get();
24 | 
25 |         final Elements tables = doc.select("div .row .col-xs-12");
26 | 
27 |         final List<ProxyDTO> proxies = tables.stream().map(table -> extractProxies(table)).flatMap(l -> l.stream())
28 |                 .collect(Collectors.toList());
29 | 
30 |         return proxies;
31 |     }
32 | 
33 |     private List<ProxyDTO> extractProxies(Element table) {
34 |         return Arrays.stream(table.text().split("\\s")).map(s -> {
35 |             final String[] token = s.split(":");
36 |             ProxyDTO proxy = new ProxyDTO();
37 |             proxy.setType("http");
38 |             proxy.setHost(token[0]);
39 |             proxy.setPort(Integer.valueOf(token[1]));
40 |             proxy.setSource("HAOIP");
41 | 
42 |             return proxy;
43 | 
44 |         }).collect(Collectors.toList());
45 |     }
46 | 
47 | }
48 | 


--------------------------------------------------------------------------------
/archive/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <parent>
 8 |         <groupId>com.mrkid.proxy</groupId>
 9 |         <artifactId>parent</artifactId>
10 |         <version>1.0-SNAPSHOT</version>
11 |     </parent>
12 | 
13 |     <groupId>com.mrkid.proxy</groupId>
14 |     <artifactId>archive</artifactId>
15 | 
16 |     <dependencies>
17 |         <dependency>
18 |             <groupId>org.springframework.boot</groupId>
19 |             <artifactId>spring-boot-starter</artifactId>
20 |         </dependency>
21 | 
22 |         <dependency>
23 |             <groupId>com.mrkid.proxy</groupId>
24 |             <artifactId>core</artifactId>
25 |         </dependency>
26 | 
27 |         <dependency>
28 |             <groupId>io.reactivex.rxjava2</groupId>
29 |             <artifactId>rxjava</artifactId>
30 | 
31 |         </dependency>
32 |         <dependency>
33 |             <groupId>org.reactivestreams</groupId>
34 |             <artifactId>reactive-streams</artifactId>
35 |         </dependency>
36 | 
37 |     </dependencies>
38 | 
39 |     <build>
40 |         <plugins>
41 |             <plugin>
42 |                 <groupId>org.springframework.boot</groupId>
43 |                 <artifactId>spring-boot-maven-plugin</artifactId>
44 |             </plugin>
45 | 
46 |         </plugins>
47 |     </build>
48 | 
49 | 
50 | 
51 | 
52 | </project>


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/ip3366/Ip3366ProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.ip3366;
 2 | 
 3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
 4 | import com.mrkid.proxy.crawler.crawler4j.ProxyWebCrawler;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 6 | import edu.uci.ics.crawler4j.crawler.Page;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.url.WebURL;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | 
13 | /**
14 |  * User: xudong
15 |  * Date: 03/11/2016
16 |  * Time: 12:47 PM
17 |  */
18 | public class Ip3366ProxyFetcher extends Crawl4jProxyFetcher {
19 | 
20 |     public static final String STORE_ROOT = "./crawl/ip3366/root";
21 |     public static final String SEED = "http://www.ip3366.net/free/";
22 | 
23 |     @Override
24 |     protected String getStoreRoot() {
25 |         return STORE_ROOT;
26 |     }
27 | 
28 |     @Override
29 |     protected List<String> getSeeds() {
30 |         return Arrays.asList(SEED);
31 |     }
32 | 
33 |     @Override
34 |     protected int getPolitenessDelay() {
35 |         return 0;
36 |     }
37 | 
38 |     @Override
39 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
40 |         return () -> new Ip3366Crawler();
41 |     }
42 | }
43 | 
44 | class Ip3366Crawler extends ProxyWebCrawler {
45 | 
46 |     public Ip3366Crawler() {
47 |         super("/xsl/ip3366.xsl", "IP3366");
48 |     }
49 | 
50 |     @Override
51 |     public boolean shouldVisit(Page referringPage, WebURL url) {
52 |         return url.getURL().startsWith("http://www.ip3366.net/free/");
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/cnproxy/CnProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.cnproxy;
 2 | 
 3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
 4 | import com.mrkid.proxy.crawler.crawler4j.ProxyWebCrawler;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 6 | import edu.uci.ics.crawler4j.crawler.Page;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.url.WebURL;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | 
13 | /**
14 |  * User: xudong
15 |  * Date: 03/11/2016
16 |  * Time: 12:47 PM
17 |  */
18 | public class CnProxyFetcher extends Crawl4jProxyFetcher {
19 | 
20 |     public static final String SEED = "http://cn-proxy.com/";
21 |     public static final String STORE_ROOT = "./crawl/cnproxy/root";
22 | 
23 |     @Override
24 |     protected String getStoreRoot() {
25 |         return STORE_ROOT;
26 |     }
27 | 
28 |     @Override
29 |     protected List<String> getSeeds() {
30 |         return Arrays.asList(SEED);
31 |     }
32 | 
33 |     @Override
34 |     protected int getPolitenessDelay() {
35 |         return 0;
36 |     }
37 | 
38 |     @Override
39 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
40 |         return () -> new CnProxyCrawler();
41 |     }
42 | }
43 | 
44 | class CnProxyCrawler extends ProxyWebCrawler {
45 |     public CnProxyCrawler() {
46 |         super("/xsl/cn-proxy.xsl", "CNPROXY");
47 |     }
48 | 
49 |     @Override
50 |     public boolean shouldVisit(Page referringPage, WebURL url) {
51 |         String anchor = url.getAnchor();
52 |         return anchor != null && anchor.equals("全球范围代理服务器");
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/kxdaili/KxDailiProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.kxdaili;
 2 | 
 3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
 4 | import com.mrkid.proxy.crawler.crawler4j.ProxyWebCrawler;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 6 | import edu.uci.ics.crawler4j.crawler.Page;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.url.WebURL;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | 
13 | /**
14 |  * User: xudong
15 |  * Date: 03/11/2016
16 |  * Time: 12:47 PM
17 |  */
18 | 
19 | public class KxDailiProxyFetcher extends Crawl4jProxyFetcher {
20 | 
21 |     public static final String STORE_ROOT = "./crawl/kxdaili/root";
22 |     public static final String SEED = "http://www.kxdaili.com/dailiip.html";
23 | 
24 |     @Override
25 |     protected String getStoreRoot() {
26 |         return STORE_ROOT;
27 |     }
28 | 
29 |     @Override
30 |     protected List<String> getSeeds() {
31 |         return Arrays.asList(SEED);
32 |     }
33 | 
34 |     @Override
35 |     protected int getPolitenessDelay() {
36 |         return 0;
37 |     }
38 | 
39 |     @Override
40 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
41 |         return () -> new KxDailiCrawler("/xsl/kxdaili.xsl", "KXDAILI");
42 |     }
43 | }
44 | 
45 | class KxDailiCrawler extends ProxyWebCrawler {
46 |     public KxDailiCrawler(String xslPath, String source) {
47 |         super(xslPath, source);
48 |     }
49 | 
50 |     @Override
51 |     public boolean shouldVisit(Page referringPage, WebURL url) {
52 |         return url.getURL().startsWith("http://www.kxdaili.com/dailiip");
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/writer/SquidFormatWriter.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.checker.writer;
 2 | 
 3 | import com.mrkid.proxy.checker.ProxyCheckResponseWriter;
 4 | import com.mrkid.proxy.dto.ProxyCheckResponse;
 5 | 
 6 | import java.io.File;
 7 | import java.io.FileWriter;
 8 | import java.io.IOException;
 9 | import java.io.PrintWriter;
10 | import java.util.Map;
11 | import java.util.concurrent.ConcurrentHashMap;
12 | 
13 | /**
14 |  * User: xudong
15 |  * Date: 16/12/2016
16 |  * Time: 5:26 PM
17 |  */
18 | public class SquidFormatWriter implements ProxyCheckResponseWriter {
19 |     final PrintWriter writer;
20 | 
21 |     private final Object object = new Object();
22 | 
23 |     /**
24 |      * same ip, diff port is not allowed in squid
25 |      */
26 |     private Map<String, Object> exportedHosts = new ConcurrentHashMap<>();
27 | 
28 |     public SquidFormatWriter(File dataDirectory) throws IOException {
29 |         writer = new PrintWriter(new FileWriter(new File(dataDirectory,
30 |                 "proxy.squid")));
31 |     }
32 | 
33 |     @Override
34 |     public boolean shouldWrite(ProxyCheckResponse response) {
35 |         return response.isValid() && "http".equalsIgnoreCase(response.getProxy().getType());
36 |     }
37 | 
38 |     @Override
39 |     public void write(ProxyCheckResponse response) {
40 |         String host = response.getProxy().getHost();
41 |         if (exportedHosts.putIfAbsent(host, object) == null) {
42 |             writer.println(
43 |                     String.format(
44 |                             "cache_peer %s parent %d 0 round-robin no-query connect-fail-limit=1 #%s %s",
45 |                             host,
46 |                             response.getProxy().getPort(),
47 |                             response.getAnonymityType().name(),
48 |                             response.getProxy().getSource()));
49 |         }
50 | 
51 |     }
52 | 
53 |     @Override
54 |     public void close() throws IOException {
55 |         writer.close();
56 | 
57 |     }
58 | }
59 | 


--------------------------------------------------------------------------------
/core/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <parent>
 8 |         <groupId>com.mrkid.proxy</groupId>
 9 |         <artifactId>parent</artifactId>
10 |         <version>1.0-SNAPSHOT</version>
11 |     </parent>
12 | 
13 |     <groupId>com.mrkid.proxy</groupId>
14 |     <artifactId>core</artifactId>
15 | 
16 |     <dependencies>
17 | 
18 |         <dependency>
19 |             <groupId>org.springframework.boot</groupId>
20 |             <artifactId>spring-boot-starter-data-jpa</artifactId>
21 |         </dependency>
22 | 
23 |         <dependency>
24 |             <groupId>org.slf4j</groupId>
25 |             <artifactId>slf4j-api</artifactId>
26 |         </dependency>
27 | 
28 |         <dependency>
29 |             <groupId>org.apache.commons</groupId>
30 |             <artifactId>commons-lang3</artifactId>
31 |         </dependency>
32 | 
33 |         <dependency>
34 |             <groupId>com.fasterxml.jackson.core</groupId>
35 |             <artifactId>jackson-databind</artifactId>
36 |         </dependency>
37 | 
38 |         <dependency>
39 |             <groupId>com.fasterxml.jackson.core</groupId>
40 |             <artifactId>jackson-core</artifactId>
41 |         </dependency>
42 | 
43 |         <dependency>
44 |             <groupId>org.projectlombok</groupId>
45 |             <artifactId>lombok</artifactId>
46 |         </dependency>
47 | 
48 |         <dependency>
49 |             <groupId>com.h2database</groupId>
50 |             <artifactId>h2</artifactId>
51 |             <version>1.4.193</version>
52 |         </dependency>
53 | 
54 |         <dependency>
55 |             <groupId>mysql</groupId>
56 |             <artifactId>mysql-connector-java</artifactId>
57 |             <version>5.1.39</version>
58 |         </dependency>
59 | 
60 |     </dependencies>
61 | 
62 | </project>


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/crawler4j/Crawl4jProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.crawler4j;
 2 | 
 3 | import com.mrkid.proxy.crawler.ProxyFetcher;
 4 | import com.mrkid.proxy.dto.ProxyDTO;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlConfig;
 6 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.fetcher.PageFetcher;
 9 | import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
10 | import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
11 | 
12 | import java.util.ArrayList;
13 | import java.util.List;
14 | 
15 | /**
16 |  * User: xudong
17 |  * Date: 12/12/2016
18 |  * Time: 8:42 AM
19 |  */
20 | public abstract class Crawl4jProxyFetcher implements ProxyFetcher {
21 |     protected abstract String getStoreRoot();
22 | 
23 |     protected abstract List<String> getSeeds();
24 | 
25 |     protected abstract int getPolitenessDelay();
26 | 
27 |     protected abstract CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory();
28 | 
29 |     public List<ProxyDTO> crawl() throws Exception {
30 | 
31 |         CrawlConfig config = new CrawlConfig();
32 |         config.setCrawlStorageFolder(getStoreRoot());
33 |         config.setPolitenessDelay(getPolitenessDelay());
34 | 
35 |         PageFetcher pageFetcher = new PageFetcher(config);
36 |         RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
37 |         RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
38 |         CrawlController crawlController = new CrawlController(config, pageFetcher, robotstxtServer);
39 | 
40 |         getSeeds().forEach(seed -> crawlController.addSeed(seed));
41 | 
42 |         int numberOfCrawlers = 1;
43 | 
44 |         crawlController.start(getWebCrawlerFactory(), numberOfCrawlers);
45 |         crawlController.waitUntilFinish();
46 |         crawlController.shutdown();
47 | 
48 |         List<ProxyDTO> result = new ArrayList<>();
49 | 
50 |         crawlController.getCrawlersLocalData().forEach(l -> ((List<ProxyDTO>) l).forEach(p -> result.add(p)));
51 | 
52 |         return result;
53 | 
54 |     }
55 | 
56 | }
57 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/p881free/P881FreeCrawler.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.p881free;
 2 | 
 3 | import com.mrkid.proxy.crawler.ProxyFetcher;
 4 | import com.mrkid.proxy.dto.ProxyDTO;
 5 | import org.apache.commons.lang3.StringUtils;
 6 | import org.jsoup.Jsoup;
 7 | import org.jsoup.nodes.Document;
 8 | import org.jsoup.nodes.Element;
 9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 | 
12 | import java.util.Arrays;
13 | import java.util.List;
14 | import java.util.stream.Collectors;
15 | 
16 | /**
17 |  * User: xudong
18 |  * Date: 03/11/2016
19 |  * Time: 12:47 PM
20 |  */
21 | public class P881FreeCrawler implements ProxyFetcher {
22 | 
23 |     private Logger logger = LoggerFactory.getLogger(P881FreeCrawler.class);
24 | 
25 |     @Override
26 |     public List<ProxyDTO> crawl() throws Exception {
27 |         Document doc = Jsoup.connect("http://881free.com/").timeout(30000).get();
28 | 
29 |         final Element body = doc.body();
30 | 
31 |         final List<ProxyDTO> proxies = Arrays.stream(body.text().split("\\s"))
32 |                 .filter(s -> StringUtils.isNotBlank(s))
33 |                 .map(s -> {
34 | 
35 |                     List<String> schemas = Arrays.asList("http://", "https://");
36 | 
37 | 
38 |                     for (String schema : schemas) {
39 |                         if (s.indexOf(schema) >= 0) {
40 |                             s = s.substring(s.indexOf(schema) + schema.length());
41 |                         }
42 |                     }
43 | 
44 |                     final String[] token = s.split(":");
45 | 
46 |                     if (token.length != 2) {
47 |                         logger.error("invalid address:" + s);
48 |                         return null;
49 |                     }
50 | 
51 |                     ProxyDTO proxy = new ProxyDTO();
52 |                     proxy.setType("http");
53 |                     proxy.setHost(token[0]);
54 |                     proxy.setPort(Integer.valueOf(token[1]));
55 |                     proxy.setSource("P881FREE");
56 | 
57 |                     return proxy;
58 | 
59 |                 }).filter(p -> p != null).collect(Collectors.toList());
60 | 
61 |         return proxies;
62 |     }
63 | }
64 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/kuaidaili/KuaiDaiLiProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.kuaidaili;
 2 | 
 3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
 4 | import com.mrkid.proxy.crawler.crawler4j.ProxyWebCrawler;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 6 | import edu.uci.ics.crawler4j.crawler.Page;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.url.WebURL;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | import java.util.regex.Matcher;
13 | import java.util.regex.Pattern;
14 | 
15 | /**
16 |  * User: xudong
17 |  * Date: 03/11/2016
18 |  * Time: 12:47 PM
19 |  */
20 | public class KuaiDaiLiProxyFetcher extends Crawl4jProxyFetcher {
21 | 
22 |     public static final String STORE_ROOT = "./crawl/kuaidaili/root";
23 |     public static final String SEED = "http://www.kuaidaili.com/free/";
24 | 
25 |     @Override
26 |     protected String getStoreRoot() {
27 |         return STORE_ROOT;
28 |     }
29 | 
30 |     @Override
31 |     protected List<String> getSeeds() {
32 |         return Arrays.asList(SEED);
33 |     }
34 | 
35 |     @Override
36 |     protected int getPolitenessDelay() {
37 |         return 2000;
38 |     }
39 | 
40 |     @Override
41 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
42 |         return () -> new KuaiDaiLiCrawler(false);
43 |     }
44 | }
45 | 
46 | class KuaiDaiLiCrawler extends ProxyWebCrawler {
47 | 
48 | 
49 |     private boolean crawlHistory = false;
50 | 
51 |     private final Pattern pagePattern = Pattern.compile("http://www.kuaidaili.com/free/[^/]+/(\\d+)/");
52 | 
53 |     public KuaiDaiLiCrawler(boolean crawlHistory) {
54 |         super("/xsl/kuaidaili.xsl", "KUAIDAILI");
55 |         this.crawlHistory = crawlHistory;
56 |     }
57 | 
58 |     @Override
59 |     public boolean shouldVisit(Page referringPage, WebURL url) {
60 |         final Matcher matcher = pagePattern.matcher(url.getURL());
61 |         if (!matcher.matches()) {
62 |             return false;
63 |         } else {
64 |             if (crawlHistory) {
65 |                 return true;
66 |             } else {
67 |                 int page = Integer.valueOf(matcher.group(1));
68 | 
69 |                 return page < 10;
70 |             }
71 |         }
72 |     }
73 | }
74 | 


--------------------------------------------------------------------------------
/archive/src/main/java/com/mrkid/proxy/archive/ArchiveApp.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.archive;
 2 | 
 3 | import com.mrkid.proxy.model.Proxy;
 4 | import com.mrkid.proxy.service.ProxyService;
 5 | import io.reactivex.Flowable;
 6 | import org.slf4j.Logger;
 7 | import org.slf4j.LoggerFactory;
 8 | import org.springframework.beans.factory.annotation.Autowired;
 9 | import org.springframework.boot.SpringApplication;
10 | import org.springframework.boot.autoconfigure.SpringBootApplication;
11 | import org.springframework.boot.autoconfigure.domain.EntityScan;
12 | import org.springframework.context.ConfigurableApplicationContext;
13 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
14 | import org.springframework.stereotype.Component;
15 | 
16 | import java.util.List;
17 | import java.util.concurrent.atomic.AtomicInteger;
18 | 
19 | /**
20 |  * User: xudong
21 |  * Date: 31/10/2016
22 |  * Time: 3:17 PM
23 |  */
24 | @SpringBootApplication(scanBasePackages = "com.mrkid.proxy")
25 | @EnableJpaRepositories("com.mrkid.proxy")
26 | @EntityScan("com.mrkid.proxy")
27 | @Component
28 | public class ArchiveApp {
29 |     @Autowired
30 |     private ProxyService proxyService;
31 | 
32 |     private Logger logger = LoggerFactory.getLogger(ArchiveApp.class);
33 | 
34 |     public static void main(String[] args) throws Exception {
35 |         final ConfigurableApplicationContext context = SpringApplication.run(ArchiveApp.class);
36 | 
37 |         context.getBean(ArchiveApp.class).archiveLowQualityProxies();
38 | 
39 |         context.close();
40 | 
41 |     }
42 | 
43 |     private void archiveLowQualityProxies() {
44 |         AtomicInteger page = new AtomicInteger(0);
45 |         int size = 1000;
46 | 
47 |         Flowable.<List<Proxy>>generate(e -> {
48 |             long start = System.currentTimeMillis();
49 |             final List<Proxy> inactiveProxies = proxyService.getInactiveProxies(page.getAndIncrement(), size);
50 |             logger.info("get page " + page.get() + " cost " + (System.currentTimeMillis() - start));
51 |             if (inactiveProxies.isEmpty()) {
52 |                 e.onComplete();
53 |             } else {
54 |                 e.onNext(inactiveProxies);
55 |             }
56 |         })
57 |                 .concatMapIterable(l -> l)
58 |                 .doOnNext(p -> proxyService.archive(p))
59 |                 .blockingSubscribe();
60 | 
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/ProxyCrawlerConfiguration.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler;
 2 | 
 3 | import com.mrkid.proxy.crawler.vender.cnproxy.CnProxyFetcher;
 4 | import com.mrkid.proxy.crawler.vender.coobobo.CooboboProxyFetcher;
 5 | import com.mrkid.proxy.crawler.vender.haoip.HaoIPCrawler;
 6 | import com.mrkid.proxy.crawler.vender.ip3366.Ip3366ProxyFetcher;
 7 | import com.mrkid.proxy.crawler.vender.kxdaili.KxDailiProxyFetcher;
 8 | import com.mrkid.proxy.crawler.vender.p66ip.P66IPProxyFetcher;
 9 | import com.mrkid.proxy.crawler.vender.p881free.P881FreeCrawler;
10 | import org.springframework.context.annotation.Bean;
11 | import org.springframework.context.annotation.Configuration;
12 | 
13 | import java.util.Arrays;
14 | import java.util.List;
15 | 
16 | /**
17 |  * User: xudong
18 |  * Date: 12/12/2016
19 |  * Time: 12:15 PM
20 |  */
21 | @Configuration
22 | public class ProxyCrawlerConfiguration {
23 | 
24 |     @Bean
25 |     public ProxyFetcher cnProxyFetcher() {
26 |         return new CnProxyFetcher();
27 |     }
28 | 
29 |     @Bean
30 |     public ProxyFetcher cooboboProxyFetcher() {
31 |         return new CooboboProxyFetcher(false);
32 |     }
33 | 
34 |     @Bean
35 |     public ProxyFetcher ip3366ProxyFetcher() {
36 |         return new Ip3366ProxyFetcher();
37 |     }
38 | 
39 |     @Bean
40 |     public ProxyFetcher kxdailiProxyFetcher() {
41 |         return new KxDailiProxyFetcher();
42 |     }
43 | 
44 |     @Bean
45 |     public ProxyFetcher p66ipProxyFetcher() {
46 |         return new P66IPProxyFetcher(false);
47 |     }
48 | 
49 |     @Bean
50 |     public ProxyFetcher p881freeProxyFetcher() {
51 |         return new P881FreeCrawler();
52 |     }
53 | 
54 |     @Bean
55 |     public ProxyFetcher haoipProxyFetcher() {
56 |         return new HaoIPCrawler();
57 |     }
58 | 
59 |     // goubanjia's proxy quality is too bad, always unavailable
60 | //    @Bean
61 | //    public ProxyFetcher goubanjiaProxyFetcher() {
62 | //        return new GoubanjiaProxyFetcher();
63 | //    }
64 | 
65 | 
66 |     // kuaidaili is sensitive to high frequency visit. proxy quality is too bad, always unavailable
67 | //    @Bean
68 | //    public ProxyFetcher kuaidailiProxyFetcher() {
69 | //        return new KuaiDaiLiProxyFetcher();
70 | //    }
71 | 
72 |     @Bean
73 |     public List<ProxyFetcher> proxyFetchers(ProxyFetcher... fetchers) {
74 |         return Arrays.asList(fetchers);
75 |     }
76 | 
77 | }
78 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/p66ip/P66IPProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.p66ip;
 2 | 
 3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
 4 | import com.mrkid.proxy.crawler.crawler4j.ProxyWebCrawler;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 6 | import edu.uci.ics.crawler4j.crawler.Page;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.url.WebURL;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | import java.util.regex.Matcher;
13 | import java.util.regex.Pattern;
14 | 
15 | /**
16 |  * User: xudong
17 |  * Date: 03/11/2016
18 |  * Time: 12:47 PM
19 |  */
20 | public class P66IPProxyFetcher extends Crawl4jProxyFetcher {
21 | 
22 |     public static final String STORE_ROOT = "./crawl/p66ip/root";
23 |     public static final String SEED = "http://www.66ip.cn/1.html";
24 | 
25 |     private boolean crawlHistory = false;
26 | 
27 |     @Override
28 |     protected String getStoreRoot() {
29 |         return STORE_ROOT;
30 |     }
31 | 
32 |     @Override
33 |     protected List<String> getSeeds() {
34 |         return Arrays.asList(SEED);
35 |     }
36 | 
37 |     @Override
38 |     protected int getPolitenessDelay() {
39 |         return 0;
40 |     }
41 | 
42 |     @Override
43 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
44 |         return () -> new P66IPCrawler(crawlHistory);
45 |     }
46 | 
47 |     public P66IPProxyFetcher(boolean crawlHistory) {
48 |         this.crawlHistory = crawlHistory;
49 |     }
50 | }
51 | 
52 | 
53 | class P66IPCrawler extends ProxyWebCrawler {
54 | 
55 |     private boolean crawlHistory = false;
56 | 
57 |     private final Pattern pagePattern = Pattern.compile("http://www.66ip.cn/(\\d+).html");
58 | 
59 |     public P66IPCrawler(boolean crawlHistory) {
60 |         super("/xsl/p66ip.xsl", "P66IP");
61 |         this.crawlHistory = crawlHistory;
62 |     }
63 | 
64 |     @Override
65 |     public boolean shouldVisit(Page referringPage, WebURL url) {
66 |         final Matcher matcher = pagePattern.matcher(url.getURL());
67 |         if (!matcher.matches()) {
68 |             return false;
69 |         } else {
70 | 
71 |             if (crawlHistory) {
72 |                 return true;
73 |             } else {
74 |                 int page = Integer.valueOf(matcher.group(1));
75 | 
76 |                 return page < 10;
77 |             }
78 |         }
79 |     }
80 | 
81 | 
82 | }
83 | 


--------------------------------------------------------------------------------
/etl/src/main/java/com/mrkid/proxy/etl/EtlApp.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.etl;
 2 | 
 3 | import com.mrkid.proxy.dto.ProxyDTO;
 4 | import com.mrkid.proxy.service.ProxyService;
 5 | import org.apache.commons.lang3.StringUtils;
 6 | import org.springframework.beans.factory.annotation.Autowired;
 7 | import org.springframework.boot.SpringApplication;
 8 | import org.springframework.boot.autoconfigure.SpringBootApplication;
 9 | import org.springframework.boot.autoconfigure.domain.EntityScan;
10 | import org.springframework.context.ConfigurableApplicationContext;
11 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
12 | import org.springframework.stereotype.Component;
13 | 
14 | import java.io.BufferedReader;
15 | import java.io.File;
16 | import java.io.FileReader;
17 | import java.io.IOException;
18 | 
19 | /**
20 |  * User: xudong
21 |  * Date: 31/10/2016
22 |  * Time: 3:17 PM
23 |  */
24 | @SpringBootApplication(scanBasePackages = "com.mrkid.proxy")
25 | @EnableJpaRepositories("com.mrkid.proxy")
26 | @EntityScan("com.mrkid.proxy")
27 | @Component
28 | public class EtlApp {
29 |     @Autowired
30 |     private ProxyService proxyService;
31 | 
32 |     public static void main(String[] args) throws Exception {
33 |         if (args.length != 2) {
34 |             System.out.println("java EtlApp [file] [source]");
35 |             return;
36 |         }
37 | 
38 |         File file = new File(args[0]);
39 | 
40 |         String source = args[1];
41 | 
42 |         final ConfigurableApplicationContext context = SpringApplication.run(EtlApp.class);
43 | 
44 |         context.getBean(EtlApp.class).etl(file, source);
45 | 
46 |         context.close();
47 | 
48 |     }
49 | 
50 |     private void etl(File file, String source) throws IOException {
51 |         try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
52 |             String line = null;
53 |             while ((line = reader.readLine()) != null) {
54 |                 if (StringUtils.isNotBlank(line)) {
55 |                     String[] tokens = line.split(":");
56 |                     String host = tokens[0];
57 |                     int port = Integer.valueOf(tokens[1]);
58 | 
59 |                     ProxyDTO proxy = new ProxyDTO();
60 |                     proxy.setHost(host);
61 |                     proxy.setPort(port);
62 |                     proxy.setSource(source);
63 | 
64 |                     proxyService.saveProxy(proxy);
65 |                 }
66 |             }
67 |         }
68 |     }
69 | }
70 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # proxy-checker
 2 | It will 
 3 | 
 4 | * crawl proxies from www.coobobo.com, cn-proxy.com, kxdaili.com, 881free.com, haoip.cc, , ip3366.net, www.66ip.cn.  
 5 |   the free proxies of www.goubanjia.com and www.kuaidaili.com are always unavailable, they are removed.   
 6 |  you can also add other proxy website as you like.
 7 | * check proxies availability
 8 | * figure out their type
 9 |   * TRANSPARENT_PROXY
10 |   * ANONYMOUS_PROXY
11 |   * DISTORTING_PROXY
12 |   * HIGH_ANONYMITY_PROXY
13 | * output proxies in squid format
14 | * Generally, you will get around ~500 avaiable proxies, which contain ~200 high anonymity proxies.
15 | 
16 | ## usage
17 | `mvn clean -Dmaven.test.skip package`
18 |  
19 | * run crawler & checker client
20 |  
21 |     * run crawler   
22 |     
23 | 			java -jar crawler/target/crawler-1.0-SNAPSHOT.jar
24 | 
25 | 
26 |     * run checker
27 |     
28 |         	java -jar checker/target/checker-1.0-SNAPSHOT.jar
29 |     
30 |      Valid proxies are exported to ./data/ in plain/squid/... format.
31 |      
32 |      By default, the check concurrency is 1000, you can increase concurrency by modifying ProxyCheckerConfiguration.maxConcurrency. 
33 |      
34 |      If you want high concurrency, do remember to update linux file-max, otherwise it will crash because of *too many open files* problem.
35 |      
36 |     * run them both
37 |     
38 |     
39 |     		java -jar crawler/target/crawler-1.0-SNAPSHOT.jar         
40 |      		java -jar checker/target/checker-1.0-SNAPSHOT.jar
41 | 
42 | 
43 | 
44 | * By default, h2base is used, and data is stored at ./db/proxy.mv.db  
45 | 
46 |   You can switch to mysql by updating core/src/main/resources/application.properties.
47 |   
48 | * The *etl* is used to batch import proxies manually.
49 |       
50 |    `java -jar etl/target/etl-1.0-SNAPSHOT.jar [filepath] [source_name]`
51 |    
52 | * The *archive* is used to archive inactive proxies. 
53 | 
54 |   If a proxy fail to check 3 times recently, it is considered as inactive.
55 |  
56 | ## Docker
57 | 
58 | It's convenient to run the proxy server in docker.
59 | 
60 | * Install docker
61 | * `mvn clean -Dmaven.test.skip package`
62 | * `docker build -t auto-proxy .`
63 | * (optional) save/load the docker image
64 | * `docker run -d -p 3128:3128 --name wifiyou-proxy auto-proxy`
65 | * `docker exec wifiyou-proxy ./update`
66 | * 127.0.0.1:3128 is ready to use as proxy server
67 | * Run `docker exec wifiyou-proxy ./update` anytime to update proxy list
68 | * Run `docker stop wifiyou-proxy` and `docker start wifiyou-proxy` to stop/start
69 | 


--------------------------------------------------------------------------------
/crawler/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>com.mrkid.proxy</groupId>
 8 |     <artifactId>crawler</artifactId>
 9 | 
10 |     <parent>
11 |         <groupId>com.mrkid.proxy</groupId>
12 |         <artifactId>parent</artifactId>
13 |         <version>1.0-SNAPSHOT</version>
14 |     </parent>
15 | 
16 |     <dependencies>
17 |         <dependency>
18 |             <groupId>com.mrkid.proxy</groupId>
19 |             <artifactId>core</artifactId>
20 |         </dependency>
21 | 
22 |         <dependency>
23 |             <groupId>org.springframework.boot</groupId>
24 |             <artifactId>spring-boot-starter</artifactId>
25 |         </dependency>
26 | 
27 |         <dependency>
28 |             <groupId>org.springframework.boot</groupId>
29 |             <artifactId>spring-boot-starter-logging</artifactId>
30 |         </dependency>
31 | 
32 |         <dependency>
33 |             <groupId>org.apache.commons</groupId>
34 |             <artifactId>commons-lang3</artifactId>
35 |         </dependency>
36 | 
37 |         <dependency>
38 |             <groupId>commons-io</groupId>
39 |             <artifactId>commons-io</artifactId>
40 |         </dependency>
41 | 
42 |         <dependency>
43 |             <groupId>com.fasterxml.jackson.core</groupId>
44 |             <artifactId>jackson-databind</artifactId>
45 |         </dependency>
46 | 
47 |         <dependency>
48 |             <groupId>com.fasterxml.jackson.core</groupId>
49 |             <artifactId>jackson-core</artifactId>
50 |         </dependency>
51 | 
52 |         <dependency>
53 |             <groupId>edu.uci.ics</groupId>
54 |             <artifactId>crawler4j</artifactId>
55 |         </dependency>
56 | 
57 |         <dependency>
58 |             <groupId>org.jsoup</groupId>
59 |             <artifactId>jsoup</artifactId>
60 |         </dependency>
61 | 
62 |         <dependency>
63 |             <groupId>junit</groupId>
64 |             <artifactId>junit</artifactId>
65 |         </dependency>
66 | 
67 |         <dependency>
68 |             <groupId>xalan</groupId>
69 |             <artifactId>xalan</artifactId>
70 |         </dependency>
71 | 
72 | 
73 |     </dependencies>
74 | 
75 |     <build>
76 |         <plugins>
77 |             <plugin>
78 |                 <groupId>org.springframework.boot</groupId>
79 |                 <artifactId>spring-boot-maven-plugin</artifactId>
80 |             </plugin>
81 | 
82 |         </plugins>
83 |     </build>
84 | 
85 | 
86 | </project>


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/coobobo/CooboboProxyFetcher.java:
--------------------------------------------------------------------------------
 1 | package com.mrkid.proxy.crawler.vender.coobobo;
 2 | 
 3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
 4 | import com.mrkid.proxy.crawler.crawler4j.ProxyWebCrawler;
 5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
 6 | import edu.uci.ics.crawler4j.crawler.Page;
 7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
 8 | import edu.uci.ics.crawler4j.url.WebURL;
 9 | 
10 | import java.util.Arrays;
11 | import java.util.List;
12 | import java.util.regex.Pattern;
13 | 
14 | /**
15 |  * User: xudong
16 |  * Date: 03/11/2016
17 |  * Time: 12:47 PM
18 |  */
19 | public class CooboboProxyFetcher extends Crawl4jProxyFetcher {
20 | 
21 |     public static final String STORE_ROOT = "./crawl/coobobo/root";
22 |     public static final String SEED = "http://www.coobobo.com/free-http-proxy/";
23 | 
24 |     private boolean crawlHistory = false;
25 | 
26 |     @Override
27 |     protected String getStoreRoot() {
28 |         return STORE_ROOT;
29 |     }
30 | 
31 |     @Override
32 |     protected List<String> getSeeds() {
33 |         return Arrays.asList(SEED);
34 |     }
35 | 
36 |     @Override
37 |     protected int getPolitenessDelay() {
38 |         return 0;
39 |     }
40 | 
41 |     @Override
42 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
43 |         return () -> new CooboboCrawler(crawlHistory);
44 |     }
45 | 
46 |     public CooboboProxyFetcher(boolean crawlHistory) {
47 |         this.crawlHistory = crawlHistory;
48 |     }
49 | }
50 | 
51 | class CooboboCrawler extends ProxyWebCrawler {
52 | 
53 |     private boolean crawlHistory = false;
54 | 
55 |     private static final String HISTROY_LIST_URL = "http://www.coobobo.com/free-http-proxy-everyday";
56 | 
57 |     private static final Pattern HISTROY_URL_PATTERN = Pattern.compile(
58 |             "http://www.coobobo.com/free-http-proxy/(\\d){4}-(\\d){2}-(\\d){2}");
59 | 
60 |     private static final Pattern LATEST_URL_PATTERN = Pattern.compile(
61 |             "http://www.coobobo.com/free-http-proxy/(\\d)+");
62 | 
63 | 
64 |     public CooboboCrawler(boolean crawlHistory) {
65 |         super("/xsl/coobobo.xsl", "COOBOBO");
66 |         this.crawlHistory = crawlHistory;
67 |     }
68 | 
69 |     @Override
70 |     public boolean shouldVisit(Page referringPage, WebURL url) {
71 |         if (url.getURL().equals(HISTROY_LIST_URL)) {
72 |             return true;
73 |         }
74 | 
75 |         if (LATEST_URL_PATTERN.matcher(url.getURL()).matches()) {
76 |             return true;
77 |         }
78 | 
79 | 
80 |         if (crawlHistory) {
81 |             return HISTROY_URL_PATTERN.matcher(url.getURL()).matches();
82 |         }
83 | 
84 |         return false;
85 |     }
86 | }
87 | 


--------------------------------------------------------------------------------
/checker/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <parent>
 8 |         <groupId>com.mrkid.proxy</groupId>
 9 |         <artifactId>parent</artifactId>
10 |         <version>1.0-SNAPSHOT</version>
11 |     </parent>
12 | 
13 |     <groupId>com.mrkid.proxy</groupId>
14 |     <artifactId>checker</artifactId>
15 | 
16 |     <dependencies>
17 |         <dependency>
18 |             <groupId>com.mrkid.proxy</groupId>
19 |             <artifactId>core</artifactId>
20 |         </dependency>
21 | 
22 |         <dependency>
23 |             <groupId>org.springframework.boot</groupId>
24 |             <artifactId>spring-boot-starter</artifactId>
25 |         </dependency>
26 | 
27 |         <dependency>
28 |             <groupId>org.springframework.boot</groupId>
29 |             <artifactId>spring-boot-starter-logging</artifactId>
30 |         </dependency>
31 | 
32 |         <dependency>
33 |             <groupId>org.apache.commons</groupId>
34 |             <artifactId>commons-lang3</artifactId>
35 |         </dependency>
36 | 
37 |         <dependency>
38 |             <groupId>commons-io</groupId>
39 |             <artifactId>commons-io</artifactId>
40 |         </dependency>
41 | 
42 |         <dependency>
43 |             <groupId>org.apache.httpcomponents</groupId>
44 |             <artifactId>httpasyncclient</artifactId>
45 |         </dependency>
46 | 
47 |         <dependency>
48 |             <groupId>io.reactivex.rxjava2</groupId>
49 |             <artifactId>rxjava</artifactId>
50 | 
51 |         </dependency>
52 |         <dependency>
53 |             <groupId>org.reactivestreams</groupId>
54 |             <artifactId>reactive-streams</artifactId>
55 |         </dependency>
56 | 
57 |         <dependency>
58 |             <groupId>org.jsoup</groupId>
59 |             <artifactId>jsoup</artifactId>
60 |         </dependency>
61 | 
62 |         <dependency>
63 |             <groupId>com.fasterxml.jackson.core</groupId>
64 |             <artifactId>jackson-databind</artifactId>
65 |         </dependency>
66 | 
67 |         <dependency>
68 |             <groupId>com.fasterxml.jackson.core</groupId>
69 |             <artifactId>jackson-core</artifactId>
70 |         </dependency>
71 | 
72 |         <dependency>
73 |             <groupId>junit</groupId>
74 |             <artifactId>junit</artifactId>
75 |         </dependency>
76 | 
77 |     </dependencies>
78 | 
79 |     <build>
80 |         <plugins>
81 |             <plugin>
82 |                 <groupId>org.springframework.boot</groupId>
83 |                 <artifactId>spring-boot-maven-plugin</artifactId>
84 |             </plugin>
85 |         </plugins>
86 |     </build>
87 | 
88 | 
89 | </project>


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/crawler4j/ProxyWebCrawler.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.crawler.crawler4j;
  2 | 
  3 | import com.mrkid.proxy.dto.ProxyDTO;
  4 | import com.mrkid.proxy.dto.ProxyListDTO;
  5 | import edu.uci.ics.crawler4j.crawler.Page;
  6 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
  7 | import edu.uci.ics.crawler4j.parser.HtmlParseData;
  8 | import org.jsoup.Jsoup;
  9 | import org.jsoup.helper.W3CDom;
 10 | import org.jsoup.nodes.Document;
 11 | import org.slf4j.Logger;
 12 | import org.slf4j.LoggerFactory;
 13 | 
 14 | import javax.xml.bind.JAXBContext;
 15 | import javax.xml.bind.Unmarshaller;
 16 | import javax.xml.transform.Transformer;
 17 | import javax.xml.transform.TransformerFactory;
 18 | import javax.xml.transform.dom.DOMSource;
 19 | import javax.xml.transform.stream.StreamResult;
 20 | import javax.xml.transform.stream.StreamSource;
 21 | import java.io.StringReader;
 22 | import java.io.StringWriter;
 23 | import java.io.Writer;
 24 | import java.util.ArrayList;
 25 | import java.util.List;
 26 | 
 27 | /**
 28 |  * User: xudong
 29 |  * Date: 15/12/2016
 30 |  * Time: 11:47 AM
 31 |  */
 32 | public class ProxyWebCrawler extends WebCrawler {
 33 |     private List<ProxyDTO> result = new ArrayList<>();
 34 | 
 35 |     private String xslPath;
 36 | 
 37 |     private String source;
 38 | 
 39 |     private Logger logger = LoggerFactory.getLogger(ProxyWebCrawler.class);
 40 | 
 41 |     public ProxyWebCrawler(String xslPath, String source) {
 42 |         this.xslPath = xslPath;
 43 |         this.source = source;
 44 |     }
 45 | 
 46 |     /**
 47 |      * This function is called when a page is fetched and ready
 48 |      * to be processed by your program.
 49 |      */
 50 |     @Override
 51 |     public void visit(Page page) {
 52 |         String url = page.getWebURL().getURL();
 53 |         System.out.println("URL: " + url);
 54 | 
 55 |         try {
 56 |             if (page.getParseData() instanceof HtmlParseData) {
 57 |                 HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
 58 | 
 59 |                 final Document document = Jsoup.parse(htmlParseData.getHtml());
 60 | 
 61 |                 Transformer transformer = TransformerFactory.newInstance()
 62 |                         .newTransformer(new StreamSource(getClass().getResourceAsStream(xslPath)));
 63 | 
 64 |                 Writer write = new StringWriter();
 65 |                 transformer.transform(new DOMSource(new W3CDom().fromJsoup(document)), new StreamResult(write));
 66 | 
 67 |                 String xml = write.toString();
 68 | 
 69 |                 // XML 1.1
 70 |                 // [#x1-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
 71 |                 String xml11pattern = "[^"
 72 |                         + "\u0001-\uD7FF"
 73 |                         + "\uE000-\uFFFD"
 74 |                         + "\ud800\udc00-\udbff\udfff"
 75 |                         + "]+";
 76 | 
 77 |                 // remove invalid character in xml
 78 |                 xml = xml.replaceAll(xml11pattern, "");
 79 | 
 80 |                 JAXBContext jc = JAXBContext.newInstance(ProxyListDTO.class);
 81 | 
 82 |                 Unmarshaller unmarshaller = jc.createUnmarshaller();
 83 |                 ProxyListDTO data = (ProxyListDTO) unmarshaller.unmarshal(new StringReader(xml));
 84 |                 data.getProxies().forEach(p -> p.setSource(source));
 85 | 
 86 | 
 87 |                 result.addAll(data.getProxies());
 88 |             }
 89 |         } catch (Exception e) {
 90 |             logger.error("fail to visit " + url, e);
 91 |         }
 92 |     }
 93 | 
 94 | 
 95 |     @Override
 96 |     public List<ProxyDTO> getMyLocalData() {
 97 |         return result;
 98 |     }
 99 | 
100 | }
101 | 


--------------------------------------------------------------------------------
/core/src/main/java/com/mrkid/proxy/service/ProxyService.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.service;
  2 | 
  3 | import com.mrkid.proxy.dto.ProxyCheckResponse;
  4 | import com.mrkid.proxy.dto.ProxyDTO;
  5 | import com.mrkid.proxy.model.LowQualityProxy;
  6 | import com.mrkid.proxy.model.Proxy;
  7 | import com.mrkid.proxy.model.ProxyKey;
  8 | import com.mrkid.proxy.repository.LowQualityProxyRepository;
  9 | import com.mrkid.proxy.repository.ProxyRepository;
 10 | import org.springframework.beans.BeanUtils;
 11 | import org.springframework.beans.factory.annotation.Autowired;
 12 | import org.springframework.data.domain.PageRequest;
 13 | import org.springframework.stereotype.Component;
 14 | import org.springframework.transaction.annotation.Transactional;
 15 | 
 16 | import java.util.Date;
 17 | import java.util.List;
 18 | 
 19 | /**
 20 |  * User: xudong
 21 |  * Date: 12/12/2016
 22 |  * Time: 1:40 PM
 23 |  */
 24 | @Component
 25 | public class ProxyService {
 26 | 
 27 |     @Autowired
 28 |     private ProxyRepository proxyRepository;
 29 | 
 30 |     @Autowired
 31 |     private LowQualityProxyRepository lowQualityProxyRepository;
 32 | 
 33 |     private final int failTimeLimit = 3;
 34 | 
 35 | 
 36 |     @Transactional
 37 |     public Proxy saveProxy(ProxyDTO proxyDTO) {
 38 |         ProxyKey key = new ProxyKey(proxyDTO.getType(), proxyDTO.getHost(), proxyDTO.getPort());
 39 |         Proxy proxy = proxyRepository.findOne(key);
 40 |         if (proxy == null) {
 41 |             proxy = new Proxy();
 42 |             proxy.setKey(key);
 43 | 
 44 |             proxy.setSource(proxyDTO.getSource());
 45 |             proxy.setLocation(proxyDTO.getLocation());
 46 |         } else {
 47 |             proxy.setSource(proxyDTO.getSource());
 48 |             proxy.setLocation(proxyDTO.getLocation());
 49 |         }
 50 | 
 51 |         return proxyRepository.save(proxy);
 52 |     }
 53 | 
 54 |     @Transactional
 55 |     public Proxy saveProxyCheckResponse(ProxyCheckResponse proxyCheckResponse) {
 56 | 
 57 |         ProxyDTO proxyDTO = proxyCheckResponse.getProxy();
 58 |         ProxyKey key = new ProxyKey(proxyDTO.getType(), proxyDTO.getHost(), proxyDTO.getPort());
 59 | 
 60 |         Proxy proxy = proxyRepository.getOne(key);
 61 | 
 62 |         proxy.setValid(proxyCheckResponse.isValid());
 63 |         proxy.setAnonymityType(proxyCheckResponse.getAnonymityType().getKey());
 64 | 
 65 |         if (proxyCheckResponse.isValid()) {
 66 |             proxy.setLastCheckSuccess(new Date());
 67 |         } else {
 68 |             proxy.setLastCheckFail(new Date());
 69 |             proxy.setRecentFailTimes(proxy.getRecentFailTimes() + 1);
 70 |         }
 71 | 
 72 |         return proxyRepository.save(proxy);
 73 |     }
 74 | 
 75 |     @Transactional
 76 |     public List<Proxy> getValidProxies() {
 77 |         return proxyRepository.findByValidIsTrue();
 78 | 
 79 |     }
 80 | 
 81 |     @Transactional
 82 |     public List<Proxy> getCheckableProxies(int page, int size) {
 83 |         return proxyRepository.findByValidIsTrueOrRecentFailTimesLessThanOrderByKeyHost(failTimeLimit, new PageRequest
 84 |                 (page, size));
 85 |     }
 86 | 
 87 | 
 88 |     @Transactional
 89 |     public List<Proxy> getInactiveProxies(int page, int size) {
 90 |         return proxyRepository.findByValidIsFalseAndRecentFailTimesGreaterThanEqualOrderByKeyHost(failTimeLimit, new
 91 |                 PageRequest(page, size));
 92 |     }
 93 | 
 94 |     @Transactional
 95 |     public void archive(Proxy proxy) {
 96 |         proxyRepository.delete(proxy);
 97 | 
 98 |         LowQualityProxy lowQualityProxy = lowQualityProxyRepository.findOne(proxy.getKey());
 99 | 
100 |         if (lowQualityProxy == null) {
101 |             lowQualityProxy = new LowQualityProxy();
102 |             BeanUtils.copyProperties(proxy, lowQualityProxy);
103 | 
104 |             lowQualityProxyRepository.save(lowQualityProxy);
105 |         }
106 |     }
107 | 
108 | 
109 | }
110 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/ProxyCrawlerApp.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.crawler;
  2 | 
  3 | import java.util.HashSet;
  4 | import java.util.List;
  5 | import java.util.concurrent.BlockingQueue;
  6 | import java.util.concurrent.LinkedBlockingQueue;
  7 | import java.util.stream.Collectors;
  8 | 
  9 | import javax.script.ScriptEngine;
 10 | import javax.script.ScriptEngineManager;
 11 | import javax.script.ScriptException;
 12 | 
 13 | import org.apache.commons.lang3.StringUtils;
 14 | import org.slf4j.Logger;
 15 | import org.slf4j.LoggerFactory;
 16 | import org.springframework.beans.factory.annotation.Autowired;
 17 | import org.springframework.boot.SpringApplication;
 18 | import org.springframework.boot.autoconfigure.SpringBootApplication;
 19 | import org.springframework.boot.autoconfigure.domain.EntityScan;
 20 | import org.springframework.context.ConfigurableApplicationContext;
 21 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 22 | import org.springframework.stereotype.Component;
 23 | 
 24 | import com.mrkid.proxy.dto.ProxyDTO;
 25 | import com.mrkid.proxy.service.ProxyService;
 26 | 
 27 | /**
 28 |  * User: xudong
 29 |  * Date: 31/10/2016
 30 |  * Time: 3:17 PM
 31 |  */
 32 | @SpringBootApplication(scanBasePackages = "com.mrkid.proxy")
 33 | @EnableJpaRepositories("com.mrkid.proxy")
 34 | @EntityScan("com.mrkid.proxy")
 35 | @Component
 36 | public class ProxyCrawlerApp {
 37 | 
 38 |     @Autowired
 39 |     private ProxyService proxyService;
 40 | 
 41 |     @Autowired
 42 |     private List<ProxyFetcher> proxyFetchers;
 43 | 
 44 |     private static final Logger logger = LoggerFactory.getLogger(ProxyCrawlerApp.class);
 45 | 
 46 |     private void crawl() {
 47 |         BlockingQueue<ProxyDTO> proxies = new LinkedBlockingQueue<>();
 48 |         ScriptEngineManager factory = new ScriptEngineManager();
 49 |         ScriptEngine engine = factory.getEngineByName("JavaScript");
 50 | 
 51 |         List<Thread> threads = proxyFetchers.stream().map(fetcher -> new Thread(() -> {
 52 |             try {
 53 |                 proxies.addAll(fetcher.crawl());
 54 |             } catch (Exception e) {
 55 |                 e.printStackTrace();
 56 |             }
 57 |         })).collect(Collectors.toList());
 58 | 
 59 | 
 60 |         threads.forEach(th -> th.start());
 61 | 
 62 |         threads.forEach(th -> {
 63 |             try {
 64 |                 th.join();
 65 |             } catch (InterruptedException e) {
 66 |                 e.printStackTrace();
 67 |             }
 68 |         });
 69 | 
 70 |         logger.info(proxies.size() + " proxies crawled");
 71 |         new HashSet<>(proxies).stream()
 72 |                 .filter(p -> StringUtils.isNotBlank(p.getHost()))
 73 |                 .forEach(p -> {
 74 |                     this.evalHostIfNeeded(engine, p);
 75 |                     proxyService.saveProxy(p);
 76 |                 });
 77 |     }
 78 |     
 79 |     /**
 80 |      * Extra post-processing to handle JS "document.write" case
 81 |      */
 82 |     private void evalHostIfNeeded(ScriptEngine engine, ProxyDTO proxy) {
 83 |         if (proxy.getHost().indexOf("document.write") != -1) {
 84 |             String result;
 85 |             try {
 86 |                 result = (String) engine.eval("var document = { write: function(s) {return s;} }; " + proxy.getHost());
 87 |                 proxy.setHost(result);
 88 |             } catch (ScriptException e) {
 89 |                 logger.warn("Failed evaling javascript on host name of proxy: " + proxy);
 90 |             }
 91 |         }
 92 |     }
 93 | 
 94 |     public static void main(String[] args) throws Exception {
 95 |         final ConfigurableApplicationContext context = SpringApplication.run(ProxyCrawlerApp.class);
 96 | 
 97 | 
 98 |         final ProxyCrawlerApp app = context.getBean(ProxyCrawlerApp.class);
 99 |         app.crawl();
100 | 
101 |         context.close();
102 | 
103 |     }
104 | 
105 | }
106 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/ProxyCheckerConfiguration.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.checker;
  2 | 
  3 | import com.mrkid.proxy.checker.utils.AddressUtils;
  4 | import com.mrkid.proxy.checker.writer.PlainFormatWriter;
  5 | import com.mrkid.proxy.checker.writer.SquidFormatWriter;
  6 | import org.apache.commons.lang3.concurrent.BasicThreadFactory;
  7 | import org.apache.http.client.config.RequestConfig;
  8 | import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
  9 | import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 10 | import org.apache.http.impl.nio.reactor.IOReactorConfig;
 11 | import org.springframework.context.annotation.Bean;
 12 | import org.springframework.context.annotation.Configuration;
 13 | 
 14 | import java.io.File;
 15 | import java.io.IOException;
 16 | import java.util.Arrays;
 17 | import java.util.List;
 18 | import java.util.concurrent.Executors;
 19 | import java.util.concurrent.ScheduledExecutorService;
 20 | import java.util.concurrent.ThreadFactory;
 21 | 
 22 | /**
 23 |  * User: xudong
 24 |  * Date: 12/12/2016
 25 |  * Time: 12:15 PM
 26 |  */
 27 | @Configuration
 28 | public class ProxyCheckerConfiguration {
 29 |     @Bean
 30 |     public CloseableHttpAsyncClient httpAsyncClient(int maxConcurrency) {
 31 | 
 32 |         final int TIMEOUT = 30 * 1000;
 33 |         // reactor config
 34 |         IOReactorConfig reactorConfig = IOReactorConfig.custom()
 35 |                 .setConnectTimeout(TIMEOUT)
 36 |                 .setSoTimeout(TIMEOUT).build();
 37 | 
 38 |         HttpAsyncClientBuilder asyncClientBuilder = HttpAsyncClientBuilder.create();
 39 |         asyncClientBuilder.setDefaultIOReactorConfig(reactorConfig);
 40 | 
 41 |         RequestConfig config = RequestConfig.custom()
 42 |                 .setConnectTimeout(TIMEOUT)
 43 |                 .setConnectionRequestTimeout(TIMEOUT)
 44 |                 .setSocketTimeout(TIMEOUT).build();
 45 |         asyncClientBuilder.setDefaultRequestConfig(config);
 46 | 
 47 |         // make sure it is less than  num in /proc/sys/fs/file-max
 48 |         // because we use several websites to determine if proxy is available,
 49 |         // the MaxConnPerRoute should be maxConcurrency * website_numbers(10 is enough)
 50 |         asyncClientBuilder.setMaxConnPerRoute(10 * maxConcurrency).setMaxConnTotal(10 * maxConcurrency);
 51 | 
 52 |         asyncClientBuilder.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, " +
 53 |                 "like Gecko) Chrome/54.0.2840.98 Safari/537.36");
 54 | 
 55 |         final CloseableHttpAsyncClient httpAsyncClient = asyncClientBuilder.build();
 56 |         httpAsyncClient.start();
 57 |         return httpAsyncClient;
 58 |     }
 59 | 
 60 | 
 61 |     @Bean
 62 |     public File dataDirectory() {
 63 |         File dataDirectory = new File("data");
 64 |         if (!dataDirectory.exists()) {
 65 |             dataDirectory.mkdir();
 66 |         }
 67 | 
 68 |         return dataDirectory;
 69 |     }
 70 | 
 71 |     @Bean
 72 |     public ProxyCheckResponseWriter squidFormatWriter(File dataDirectory) throws IOException {
 73 |         return new SquidFormatWriter(dataDirectory);
 74 |     }
 75 | 
 76 |     @Bean
 77 |     public ProxyCheckResponseWriter plainFormatWriter(File dataDirectory) throws IOException {
 78 |         return new PlainFormatWriter(dataDirectory);
 79 |     }
 80 | 
 81 |     @Bean
 82 |     public List<ProxyCheckResponseWriter> proxyCheckResponseWriters(ProxyCheckResponseWriter... writers) {
 83 |         return Arrays.asList(writers);
 84 |     }
 85 | 
 86 | 
 87 |     @Bean
 88 |     public String ip() throws IOException {
 89 |         return AddressUtils.getMyPublicIp();
 90 |     }
 91 | 
 92 |     @Bean
 93 |     public int maxConcurrency() {
 94 |         return 1000;
 95 |     }
 96 | 
 97 |     @Bean
 98 |     public int overallTimeout() {
 99 |         return 60 * 1000;
100 |     }
101 | 
102 |     @Bean
103 |     public ScheduledExecutorService scheduledExecutorService() {
104 |         return Executors.newScheduledThreadPool(10,
105 |                 new BasicThreadFactory.Builder().namingPattern("Timeout Scheduler-%d").build());
106 |     }
107 | 
108 | 
109 | }
110 | 


--------------------------------------------------------------------------------
/crawler/src/main/java/com/mrkid/proxy/crawler/vender/goubanjia/GoubanjiaProxyFetcher.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.crawler.vender.goubanjia;
  2 | 
  3 | import com.mrkid.proxy.crawler.crawler4j.Crawl4jProxyFetcher;
  4 | import com.mrkid.proxy.dto.ProxyDTO;
  5 | import edu.uci.ics.crawler4j.crawler.CrawlController;
  6 | import edu.uci.ics.crawler4j.crawler.Page;
  7 | import edu.uci.ics.crawler4j.crawler.WebCrawler;
  8 | import edu.uci.ics.crawler4j.parser.HtmlParseData;
  9 | import edu.uci.ics.crawler4j.url.WebURL;
 10 | import org.apache.commons.lang3.StringUtils;
 11 | import org.jsoup.Jsoup;
 12 | import org.jsoup.nodes.Document;
 13 | import org.jsoup.nodes.Element;
 14 | import org.jsoup.select.Elements;
 15 | 
 16 | import java.util.ArrayList;
 17 | import java.util.Arrays;
 18 | import java.util.List;
 19 | import java.util.stream.Collectors;
 20 | 
 21 | /**
 22 |  * User: xudong
 23 |  * Date: 03/11/2016
 24 |  * Time: 12:47 PM
 25 |  */
 26 | public class GoubanjiaProxyFetcher extends Crawl4jProxyFetcher {
 27 | 
 28 |     public static final String STORE_ROOT = "./crawl/goubanjia/root";
 29 |     public static final String SEED = "http://www.goubanjia.com/free/index.shtml";
 30 | 
 31 |     @Override
 32 |     protected String getStoreRoot() {
 33 |         return STORE_ROOT;
 34 |     }
 35 | 
 36 |     @Override
 37 |     protected List<String> getSeeds() {
 38 |         return Arrays.asList(SEED);
 39 |     }
 40 | 
 41 |     @Override
 42 |     protected int getPolitenessDelay() {
 43 |         return 0;
 44 |     }
 45 | 
 46 |     @Override
 47 |     protected CrawlController.WebCrawlerFactory<WebCrawler> getWebCrawlerFactory() {
 48 |         return () -> new GoubanjiaCrawler();
 49 |     }
 50 | }
 51 | 
 52 | class GoubanjiaCrawler extends WebCrawler {
 53 | 
 54 |     private List<ProxyDTO> result = new ArrayList<>();
 55 | 
 56 |     @Override
 57 |     public boolean shouldVisit(Page referringPage, WebURL url) {
 58 |         return url.getURL().startsWith("http://www.goubanjia.com/free/");
 59 |     }
 60 | 
 61 |     /**
 62 |      * This function is called when a page is fetched and ready
 63 |      * to be processed by your program.
 64 |      */
 65 |     @Override
 66 |     public void visit(Page page) {
 67 |         String url = page.getWebURL().getURL();
 68 |         System.out.println("URL: " + url);
 69 | 
 70 |         if (page.getParseData() instanceof HtmlParseData) {
 71 |             HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
 72 |             String html = htmlParseData.getHtml();
 73 | 
 74 |             final Document doc = Jsoup.parse(html);
 75 |             final Elements tables = doc.select("#list > table");
 76 | 
 77 |             final List<ProxyDTO> proxies = tables.stream().map(table -> extractProxies(table)).flatMap(l -> l.stream())
 78 |                     .collect(Collectors.toList());
 79 | 
 80 |             proxies.forEach(p -> result.add(p));
 81 |         }
 82 |     }
 83 | 
 84 |     private List<ProxyDTO> extractProxies(Element table) {
 85 |         final Elements header = table.select("thead tr th");
 86 |         final Elements rows = table.select("tbody tr");
 87 | 
 88 |         return rows.stream().map(row -> {
 89 |             final int size = header.size();
 90 | 
 91 |             String host = "";
 92 |             int port = 0;
 93 | 
 94 |             String location = null;
 95 | 
 96 |             Elements cells = row.select("td");
 97 | 
 98 |             for (int i = 0; i < size; i++) {
 99 |                 String headerName = header.get(i).text();
100 |                 switch (headerName) {
101 |                     case "IP:PORT":
102 |                         final Element address = cells.get(i);
103 | 
104 |                         final String before = address.toString();
105 |                         address.select("*[style~=display:\\s*none]").remove();
106 | 
107 |                         final String[] token = address.text().replaceAll("\\s", "").split(":");
108 |                         host = token[0];
109 |                         port = Integer.valueOf(token[1]);
110 | 
111 |                         logger.debug("extract " + host + ":" + port + " from " + before);
112 |                         break;
113 |                     case "IP归属地":
114 |                         location = cells.get(i).text();
115 |                         break;
116 |                     default:
117 |                 }
118 | 
119 |             }
120 | 
121 |             if (StringUtils.isBlank(host) || port == 0) {
122 |                 return null;
123 |             }
124 | 
125 |             ProxyDTO proxy = new ProxyDTO();
126 |             proxy.setHost(host);
127 |             proxy.setPort(port);
128 |             proxy.setType("http");
129 |             proxy.setLocation(location);
130 | 
131 |             proxy.setSource("GOUBANJIA");
132 | 
133 | 
134 |             return proxy;
135 |         }).filter(p -> p != null).collect(Collectors.toList());
136 |     }
137 | }
138 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/ProxyCheckerApp.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.checker;
  2 | 
  3 | import com.mrkid.proxy.dto.ProxyDTO;
  4 | import com.mrkid.proxy.model.Proxy;
  5 | import com.mrkid.proxy.service.ProxyService;
  6 | import io.reactivex.Flowable;
  7 | import io.reactivex.disposables.Disposable;
  8 | import io.reactivex.schedulers.Schedulers;
  9 | import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 10 | import org.slf4j.Logger;
 11 | import org.slf4j.LoggerFactory;
 12 | import org.springframework.beans.factory.annotation.Autowired;
 13 | import org.springframework.boot.SpringApplication;
 14 | import org.springframework.boot.autoconfigure.SpringBootApplication;
 15 | import org.springframework.boot.autoconfigure.domain.EntityScan;
 16 | import org.springframework.context.ConfigurableApplicationContext;
 17 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 18 | import org.springframework.stereotype.Component;
 19 | 
 20 | import java.util.List;
 21 | import java.util.concurrent.ExecutorService;
 22 | import java.util.concurrent.Executors;
 23 | import java.util.concurrent.TimeUnit;
 24 | import java.util.concurrent.atomic.AtomicInteger;
 25 | 
 26 | /**
 27 |  * User: xudong
 28 |  * Date: 31/10/2016
 29 |  * Time: 3:17 PM
 30 |  */
 31 | @SpringBootApplication(scanBasePackages = "com.mrkid.proxy")
 32 | @EnableJpaRepositories("com.mrkid.proxy")
 33 | @EntityScan("com.mrkid.proxy")
 34 | @Component
 35 | public class ProxyCheckerApp {
 36 |     @Autowired
 37 |     private ProxyChecker proxyChecker;
 38 | 
 39 |     @Autowired
 40 |     private ProxyService proxyService;
 41 | 
 42 |     @Autowired
 43 |     private Integer maxConcurrency;
 44 | 
 45 |     @Autowired
 46 |     private List<ProxyCheckResponseWriter> proxyCheckResponseWriters;
 47 | 
 48 |     private static final Logger logger = LoggerFactory.getLogger(ProxyCheckerApp.class);
 49 | 
 50 |     private Flowable<ProxyDTO> proxyGenerator() {
 51 |         AtomicInteger page = new AtomicInteger(0);
 52 |         int size = 1000;
 53 | 
 54 |         return Flowable.<List<Proxy>>generate(e -> {
 55 |             final List<Proxy> checkableProxies = proxyService.getCheckableProxies(page.getAndIncrement(), size);
 56 |             System.out.println("process page " + page.get());
 57 |             if (checkableProxies.isEmpty()) {
 58 |                 e.onComplete();
 59 |             } else {
 60 |                 e.onNext(checkableProxies);
 61 |             }
 62 |         })
 63 |                 .concatMapIterable(l -> l, 20000)
 64 |                 .map(p -> {
 65 |                     ProxyDTO proxyDTO = new ProxyDTO();
 66 |                     proxyDTO.setType(p.getKey().getType());
 67 |                     proxyDTO.setHost(p.getKey().getHost());
 68 |                     proxyDTO.setPort(p.getKey().getPort());
 69 |                     proxyDTO.setSource(p.getSource());
 70 |                     return proxyDTO;
 71 |                 });
 72 |     }
 73 | 
 74 |     private void check() {
 75 |         AtomicInteger concurrency = new AtomicInteger(0);
 76 | 
 77 |         AtomicInteger dispatchedCount = new AtomicInteger(0);
 78 | 
 79 |         Disposable audit = Flowable.interval(1, 1, TimeUnit.SECONDS).subscribe(l ->
 80 |                 System.out.println("dispatchedCount " + dispatchedCount.get() + " concurrency " + concurrency.get())
 81 |         );
 82 | 
 83 |         ExecutorService checkerExecutor = Executors.newFixedThreadPool(10,
 84 |                 new BasicThreadFactory.Builder().namingPattern("ProxyChecker-Scheduler-%d").build());
 85 | 
 86 |         ExecutorService saverExecutor = Executors.newFixedThreadPool(10,
 87 |                 new BasicThreadFactory.Builder().namingPattern("ProxySaver-Scheduler-%d").build());
 88 | 
 89 | 
 90 |         proxyGenerator()
 91 |                 .doOnNext(p -> concurrency.incrementAndGet())
 92 |                 .doOnNext(p -> dispatchedCount.incrementAndGet())
 93 |                 .flatMap(p ->
 94 |                                 Flowable.defer(() -> proxyChecker.check(p))
 95 |                                         .subscribeOn(Schedulers.from(checkerExecutor))
 96 |                         , maxConcurrency)
 97 |                 .doOnNext(p -> concurrency.decrementAndGet()).
 98 |                 doOnNext(p -> proxyCheckResponseWriters.forEach(writer -> {
 99 |                     if (writer.shouldWrite(p)) writer.write(p);
100 |                 }))
101 |                 .flatMap(p ->
102 |                         Flowable.defer(() -> Flowable.just(proxyService.saveProxyCheckResponse(p)))
103 |                                 .subscribeOn(Schedulers.from(saverExecutor)))
104 |                 .blockingSubscribe();
105 | 
106 |         audit.dispose();
107 | 
108 |         checkerExecutor.shutdown();
109 |         saverExecutor.shutdown();
110 | 
111 |     }
112 | 
113 | 
114 |     public static void main(String[] args) throws Exception {
115 |         final ConfigurableApplicationContext context = SpringApplication.run(ProxyCheckerApp.class);
116 | 
117 |         final ProxyCheckerApp app = context.getBean(ProxyCheckerApp.class);
118 |         app.check();
119 | 
120 |         context.close();
121 |     }
122 | 
123 | }
124 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  3 |     <modelVersion>4.0.0</modelVersion>
  4 |     <groupId>com.mrkid.proxy</groupId>
  5 |     <artifactId>parent</artifactId>
  6 |     <version>1.0-SNAPSHOT</version>
  7 |     <packaging>pom</packaging>
  8 | 
  9 |     <properties>
 10 |         <spring-boot.version>1.4.1.RELEASE</spring-boot.version>
 11 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 12 |     </properties>
 13 | 
 14 | 
 15 |     <modules>
 16 | 
 17 |         <module>core</module>
 18 |         <module>crawler</module>
 19 |         <module>checker</module>
 20 |         <module>etl</module>
 21 |         <module>archive</module>
 22 |     </modules>
 23 | 
 24 |     <dependencyManagement>
 25 |         <dependencies>
 26 |             <dependency>
 27 |                 <!-- Import dependency management from Spring Boot -->
 28 |                 <groupId>org.springframework.boot</groupId>
 29 |                 <artifactId>spring-boot-dependencies</artifactId>
 30 |                 <version>${spring-boot.version}</version>
 31 |                 <type>pom</type>
 32 |                 <scope>import</scope>
 33 |             </dependency>
 34 | 
 35 |             <dependency>
 36 |                 <groupId>com.mrkid.proxy</groupId>
 37 |                 <artifactId>core</artifactId>
 38 |                 <version>${project.version}</version>
 39 |             </dependency>
 40 | 
 41 |             <dependency>
 42 |                 <groupId>edu.uci.ics</groupId>
 43 |                 <artifactId>crawler4j</artifactId>
 44 |                 <version>4.2</version>
 45 |             </dependency>
 46 | 
 47 |             <dependency>
 48 |                 <groupId>org.jsoup</groupId>
 49 |                 <artifactId>jsoup</artifactId>
 50 |                 <version>1.10.1</version>
 51 |             </dependency>
 52 | 
 53 |             <dependency>
 54 |                 <groupId>com.fasterxml.jackson.core</groupId>
 55 |                 <artifactId>jackson-databind</artifactId>
 56 |                 <version>2.8.3</version>
 57 |             </dependency>
 58 | 
 59 |             <dependency>
 60 |                 <groupId>com.fasterxml.jackson.core</groupId>
 61 |                 <artifactId>jackson-core</artifactId>
 62 |                 <version>2.8.3</version>
 63 |             </dependency>
 64 | 
 65 |             <dependency>
 66 |                 <groupId>org.apache.httpcomponents</groupId>
 67 |                 <artifactId>httpasyncclient</artifactId>
 68 |                 <version>4.1.2</version>
 69 |             </dependency>
 70 | 
 71 |             <dependency>
 72 |                 <groupId>org.projectlombok</groupId>
 73 |                 <artifactId>lombok</artifactId>
 74 |                 <version>1.16.10</version>
 75 |             </dependency>
 76 | 
 77 |             <dependency>
 78 |                 <groupId>org.apache.commons</groupId>
 79 |                 <artifactId>commons-lang3</artifactId>
 80 |                 <version>3.4</version>
 81 |             </dependency>
 82 | 
 83 |             <dependency>
 84 |                 <groupId>commons-io</groupId>
 85 |                 <artifactId>commons-io</artifactId>
 86 |                 <version>2.4</version>
 87 |             </dependency>
 88 | 
 89 |             <dependency>
 90 |                 <groupId>io.reactivex.rxjava2</groupId>
 91 |                 <artifactId>rxjava</artifactId>
 92 |                 <version>2.0.1</version>
 93 | 
 94 |             </dependency>
 95 |             <dependency>
 96 |                 <groupId>org.reactivestreams</groupId>
 97 |                 <artifactId>reactive-streams</artifactId>
 98 |                 <version>1.0.0</version>
 99 |             </dependency>
100 | 
101 |             <dependency>
102 |                 <groupId>junit</groupId>
103 |                 <artifactId>junit</artifactId>
104 |                 <version>4.12</version>
105 |                 <scope>test</scope>
106 |             </dependency>
107 | 
108 |             <dependency>
109 |                 <groupId>xalan</groupId>
110 |                 <artifactId>xalan</artifactId>
111 |                 <version>2.7.2</version>
112 |             </dependency>
113 | 
114 |         </dependencies>
115 |     </dependencyManagement>
116 | 
117 |     <build>
118 |         <pluginManagement>
119 |             <plugins>
120 |                 <plugin>
121 |                     <groupId>org.springframework.boot</groupId>
122 |                     <artifactId>spring-boot-maven-plugin</artifactId>
123 |                     <version>${spring-boot.version}</version>
124 |                     <executions>
125 |                         <execution>
126 |                             <goals>
127 |                                 <goal>repackage</goal>
128 |                             </goals>
129 |                         </execution>
130 |                     </executions>
131 |                 </plugin>
132 | 
133 |                 <plugin>
134 |                     <groupId>org.projectlombok</groupId>
135 |                     <artifactId>lombok-maven-plugin</artifactId>
136 |                     <version>1.16.2.0</version>
137 |                 </plugin>
138 | 
139 |                 <plugin>
140 |                     <artifactId>maven-compiler-plugin</artifactId>
141 |                     <version>3.5.1</version>
142 |                     <configuration>
143 |                         <source>1.8</source>
144 |                         <target>1.8</target>
145 |                     </configuration>
146 |                 </plugin>
147 | 
148 |             </plugins>
149 |         </pluginManagement>
150 |     </build>
151 | 
152 | 
153 | 
154 | </project>
155 | 


--------------------------------------------------------------------------------
/checker/src/main/java/com/mrkid/proxy/checker/ProxyChecker.java:
--------------------------------------------------------------------------------
  1 | package com.mrkid.proxy.checker;
  2 | 
  3 | import java.io.IOException;
  4 | import java.net.InetSocketAddress;
  5 | import java.util.Arrays;
  6 | import java.util.HashSet;
  7 | import java.util.Iterator;
  8 | import java.util.Set;
  9 | import java.util.concurrent.CompletableFuture;
 10 | import java.util.concurrent.Future;
 11 | import java.util.concurrent.ScheduledExecutorService;
 12 | import java.util.concurrent.TimeUnit;
 13 | 
 14 | import org.apache.commons.io.IOUtils;
 15 | import org.apache.http.HttpHost;
 16 | import org.apache.http.HttpResponse;
 17 | import org.apache.http.HttpStatus;
 18 | import org.apache.http.client.config.RequestConfig;
 19 | import org.apache.http.client.methods.HttpGet;
 20 | import org.apache.http.client.methods.HttpRequestBase;
 21 | import org.apache.http.client.protocol.HttpClientContext;
 22 | import org.apache.http.concurrent.FutureCallback;
 23 | import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 24 | import org.apache.http.protocol.HttpContext;
 25 | import org.apache.http.util.TextUtils;
 26 | import org.slf4j.Logger;
 27 | import org.slf4j.LoggerFactory;
 28 | import org.springframework.beans.factory.annotation.Autowired;
 29 | import org.springframework.stereotype.Component;
 30 | 
 31 | import com.fasterxml.jackson.databind.JsonNode;
 32 | import com.fasterxml.jackson.databind.ObjectMapper;
 33 | import com.mrkid.proxy.dto.AnonymityType;
 34 | import com.mrkid.proxy.dto.ProxyCheckResponse;
 35 | import com.mrkid.proxy.dto.ProxyDTO;
 36 | 
 37 | import io.reactivex.Flowable;
 38 | import io.reactivex.processors.BehaviorProcessor;
 39 | 
 40 | /**
 41 |  * User: xudong
 42 |  * Date: 31/10/2016
 43 |  * Time: 3:39 PM
 44 |  */
 45 | @Component
 46 | public class ProxyChecker {
 47 |     private ObjectMapper objectMapper = new ObjectMapper();
 48 | 
 49 |     @Autowired
 50 |     private CloseableHttpAsyncClient httpclient;
 51 | 
 52 |     @Autowired
 53 |     private ScheduledExecutorService scheduledExecutorService;
 54 | 
 55 |     @Autowired
 56 |     private Integer overallTimeout = 0;
 57 | 
 58 |     private static final Logger logger = LoggerFactory.getLogger(ProxyChecker.class);
 59 | 
 60 |     public Flowable<ProxyCheckResponse> check(ProxyDTO proxyDTO) {
 61 | 
 62 |         long start = System.currentTimeMillis();
 63 | 
 64 |         final Flowable<ProxyCheckResponse> proxyCheckResponseFlow = getProxyResponse(proxyDTO);
 65 | 
 66 |         final Flowable<String> ip138Flow = generalGet("http://1212.ip138.com/ic.asp", proxyDTO);
 67 | 
 68 |         return Flowable.zip(proxyCheckResponseFlow, ip138Flow,
 69 |                 (response, s) -> response)
 70 |                 .onErrorResumeNext(e -> {
 71 |                     return Flowable.just(new ProxyCheckResponse("", "", proxyDTO, false));
 72 |                 }).doFinally(
 73 |                         () -> logger.info("checking {}://{}:{} takes {} ms",
 74 |                                 proxyDTO.getType(), proxyDTO.getHost(), proxyDTO.getPort(),
 75 |                                 System.currentTimeMillis() - start)
 76 |                 );
 77 |     }
 78 | 
 79 | 
 80 |     private Flowable<ProxyCheckResponse> getProxyResponse(ProxyDTO proxy) {
 81 |         final Flowable<String> remoteIpFlow = toFlowable(execute(proxy, new HttpGet("http://httpbin.org/ip")));
 82 | 
 83 |         final Flowable<String> headersFlow = toFlowable(execute(proxy, new HttpGet("http://httpbin.org/headers")));
 84 | 
 85 |         return remoteIpFlow.zipWith(headersFlow, (remoteIpResponse, headersResponse) -> {
 86 |             String remoteIp = objectMapper.readTree(remoteIpResponse).get("origin").asText();
 87 |             final JsonNode headersNode = objectMapper.readTree(headersResponse).get("headers");
 88 | 
 89 |             final Set<String> possibleHeaderNames = new HashSet<>(Arrays.asList("HTTP_VIA", "HTTP_X_FORWARDED_FOR",
 90 |                     "HTTP_FORWARDED_FOR", "HTTP_X_FORWARDED", "HTTP_FORWARDED", "HTTP_CLIENT_IP",
 91 |                     "HTTP_FORWARDED_FOR_IP", "VIA", "X_FORWARDED_FOR", "FORWARDED_FOR", "X_FORWARDED",
 92 |                     "FORWARDED", "CLIENT_IP", "FORWARDED_FOR_IP", "HTTP_PROXY_CONNECTION"));
 93 | 
 94 | 
 95 |             boolean highAnonymity = true;
 96 |             final Iterator<String> fieldNames = headersNode.fieldNames();
 97 |             while (fieldNames.hasNext()) {
 98 |                 if (possibleHeaderNames.contains(fieldNames.next().toUpperCase())) {
 99 |                     highAnonymity = false;
100 |                 }
101 |             }
102 | 
103 |             ProxyCheckResponse proxyCheckResponse = new ProxyCheckResponse();
104 |             proxyCheckResponse.setProxy(proxy);
105 |             proxyCheckResponse.setRemoteIp(remoteIp);
106 |             proxyCheckResponse.setValid(true);
107 |             if (highAnonymity) {
108 |                 proxyCheckResponse.setAnonymityType(AnonymityType.HIGH_ANONYMITY_PROXY);
109 |             } else {
110 |                 proxyCheckResponse.setAnonymityType(AnonymityType.TRANSPARENT_PROXY);
111 |             }
112 | 
113 |             return proxyCheckResponse;
114 |         });
115 | 
116 |     }
117 | 
118 |     private Flowable<String> generalGet(String targetUrl, ProxyDTO proxy) {
119 |         final HttpGet request = new HttpGet(targetUrl);
120 |         return toFlowable(execute(proxy, request));
121 |     }
122 | 
123 |     private CompletableFuture<String> execute(ProxyDTO proxy, HttpRequestBase request) {
124 | 
125 |         CompletableFuture<String> promise = new CompletableFuture<>();
126 |         
127 |         // Bugfix: If host name contains blanks, new HttpHost() below will throw exception and promise won't
128 |         // complete forever.
129 |         if (TextUtils.containsBlanks(proxy.getHost())) {
130 |             logger.warn("cancelling proxy: " + proxy + " since it contains blanks in host name");
131 |             promise.cancel(true);
132 |             return promise;
133 |         }
134 | 
135 |         HttpContext httpContext = HttpClientContext.create();
136 | 
137 |         logger.info("check proxy: " + proxy + " for url " + request.getURI().toString());
138 | 
139 |         if (proxy.getType().equalsIgnoreCase("socks5") || proxy.getType().equalsIgnoreCase("socks4")) {
140 |             httpContext.setAttribute("socks.address", new InetSocketAddress(proxy.getHost(), proxy.getPort()));
141 |         } else if (proxy.getType().equalsIgnoreCase("http") || proxy.getType().equalsIgnoreCase("https")) {
142 |             RequestConfig config = RequestConfig.custom()
143 |                     .setProxy(new HttpHost(proxy.getHost(), proxy.getPort(), proxy.getType().toLowerCase()))
144 |                     .build();
145 | 
146 |             request.setConfig(config);
147 | 
148 |         }
149 | 
150 |         Future<HttpResponse> future = httpclient.execute(request, httpContext, new FutureCallback<HttpResponse>() {
151 |             @Override
152 |             public void completed(HttpResponse httpResponse) {
153 |                 logger.info("completed proxy: " + proxy + " for url " + request.getURI().toString());
154 | 
155 |                 if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
156 |                     final String message = "status code is " + httpResponse.getStatusLine
157 |                             ().getStatusCode();
158 |                     logger.error(message);
159 |                     promise.completeExceptionally(new RuntimeException(message));
160 |                 } else {
161 |                     try {
162 |                         final String body = IOUtils.toString(httpResponse.getEntity().getContent(), "utf-8");
163 |                         logger.info("get " + body + " from " + request.getURI() + " through " + proxy);
164 |                         promise.complete(body);
165 |                     } catch (IOException e) {
166 |                         logger.error("unable to parse check response of {}, {}, {}",
167 |                                 httpResponse.getEntity(),
168 |                                 e.getClass().getName(),
169 |                                 e.getMessage());
170 | 
171 |                         promise.completeExceptionally(e);
172 |                     }
173 |                 }
174 |             }
175 | 
176 |             @Override
177 |             public void failed(Exception e) {
178 |                 logger.error("failed to visit {} through  {} caused by  {} {}",
179 |                         request.getURI().toString(), proxy, e.getClass().getName(), e.getMessage());
180 |                 promise.completeExceptionally(e);
181 |             }
182 | 
183 |             @Override
184 |             public void cancelled() {
185 |                 logger.info("cancel proxy: " + proxy + " for url " + request.getURI().toString());
186 | 
187 |                 promise.cancel(true);
188 |             }
189 |         });
190 | 
191 |         if (overallTimeout > 0) {
192 |             scheduledExecutorService.schedule(() -> {
193 |                 if (!future.isDone()) {
194 |                     future.cancel(true);
195 |                 }
196 |             }, overallTimeout, TimeUnit.MILLISECONDS);
197 |         }
198 | 
199 |         return promise;
200 |     }
201 | 
202 |     private <T> Flowable<T> toFlowable(CompletableFuture<T> future) {
203 |         final BehaviorProcessor<T> processor = BehaviorProcessor.create();
204 | 
205 |         future.whenComplete((result, error) -> {
206 |             if (error != null) {
207 |                 processor.onError(error);
208 |             } else {
209 |                 processor.onNext(result);
210 |                 processor.onComplete();
211 |             }
212 |         });
213 | 
214 | 
215 |         return processor;
216 | 
217 | 
218 |     }
219 | }
220 | 


--------------------------------------------------------------------------------
/crawler/src/test/resources/html/kxdaili.html:
--------------------------------------------------------------------------------
  1 | 
  2 | 
  3 | <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
  4 | <html>
  5 | <head>
  6 |     <title>免费代理IP - 大陆高匿代理IP - 开心代理IP平台 | 时刻在更新的专业代理IP平台</title>
  7 |     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
  8 |     <meta name="Keywords" content="代理IP|开心代理IP平台|买代理IP|提取代理IP|高质量代理IP|免费代理IP|国内ip代理|IP代理|最新代理IP|今日可用代理IP">
  9 |     <meta name="Description" content="开心代理IP平台是国内领先的代理IP平台，时时刻刻更新代理IP，代理IP有效率高达90%，每天流水10-25万代理IP，深受广大用户喜爱的代理IP平台，提取代理IP就上开心代理IP平台。">
 10 | 
 11 |     <script language="javascript" type="text/javascript" src="/public/view/default/js/jquery-1.9.1.min.js"></script>
 12 |     <link href="/public/view/default/images/favicon2.ico" rel="shortcut icon" type="image/vnd.microsoft.icon" /><link href="/public/view/default/css/css.css" rel="stylesheet" type="text/css">
 13 | </head>
 14 | <body id="nav_btn01">
 15 | <div class="header"><dl class="header">
 16 |     <dd><span><a href="/login.html" class="">登陆</a></span><a href="/reg.html" class="">注册</a></dd>
 17 | </dl></div>
 18 | <!--end头文件-->
 19 | <div class="header_c">
 20 |     <div class="header_c_w">
 21 |         <div class="logo"><a href="/"><img src="/public/view/default/images/logo.png" alt="dns"></a></div>
 22 |         <!--end标志-->
 23 |         <div class="nav th">
 24 |             <ul class="nav last_list">
 25 |                 <li id="nav_hover01"><a href="/">首页</a></li>
 26 |                 <li id="nav_hover02"  class="active"><a href="/dailiip.html">免费代理IP</a></li>
 27 |                 <li id="nav_hover03" ><a href="/buy.html">购买代理IP</a></li>
 28 |                 <li id="nav_hover04" ><a href="/tongji.html">代理IP统计</a></li>
 29 |                 <li id="nav_hover05" ><a href="/api.html" >API链接生成</a></li>
 30 |                 <li id="nav_hover06" ><a href="/lianxi.html" >联系我们</a></li>
 31 |             </ul>
 32 |         </div>
 33 |         <!--end导航-->
 34 |     </div>
 35 |     <!--end头文件宽度-->
 36 | </div>
 37 | <!--end头文件中-->
 38 | 
 39 | <script language='javascript'>
 40 | if (document.location.protocol=="http:") {
 41 | 	//http redirect to https
 42 | 	var new_url = "https://" + document.location.hostname + document.location.pathname + document.location.search;
 43 | 	//document.location = new_url;
 44 | }
 45 | </script>
 46 | <!-- 阴影 --><div class="header_c2 i_header_c"></div>
 47 | <link rel="stylesheet" type="text/css" class="ui" href="/public/view/default/css/semantic.min.css">
 48 | 
 49 | 
 50 | <div class="tab_c_box buy_tab_box" style="margin-top:40px;">
 51 |     <div class="tag_area" >
 52 |         <a id="tag_1" class="label" href="/dailiip.html">大陆高匿代理</a>
 53 |         <a id="tag_2" class="label" href="/dailiip/2/1.html">大陆普匿代理</a>
 54 |         <a id="tag_3" class="label" href="/dailiip/3/1.html">海外高匿代理</a>
 55 |         <a id="tag_4" class="label" href="/dailiip/4/1.html">海外普匿代理</a>
 56 |     </div>
 57 | </div>
 58 | 
 59 | <script type="text/javascript">
 60 |     $("#tag_1").addClass("active")
 61 | $(document).ready(function() {
 62 | });
 63 | </script>
 64 | 
 65 | 
 66 | 
 67 | 
 68 | <style>
 69 | .segment th{
 70 | 	color:#777!important;
 71 | }
 72 | .segment td{
 73 | 	font-size:14px;
 74 | 	height:25px;
 75 | 	line-height:25px;
 76 | }
 77 | </style><a name="ip"></a>
 78 | 
 79 | <div style="text-align:center;width:100%;margin-top:-20px;margin-bottom:0px;">
 80 |     <a href="/buy.html" class="buy_btn_my">想要更多、质量更好的代理IP，请点这里购买。</a></div>
 81 | </div>
 82 | 
 83 | <div class="tab_c_box buy_tab_box" style="margin-top:20px;">
 84 |     <table class="ui table segment" >
 85 |         <thead>
 86 |         <th>IP地址</th>
 87 |         <th>端口</th>
 88 |         <th>匿名等级</th>
 89 |         <th>代理类型</th>
 90 |         <th>响应时间</th>
 91 |         <th>地理位置</th>
 92 |         <th>最近验证时间</th>
 93 |         </thead>
 94 |         <tbody>
 95 |         <tr>
 96 |             <td>27.19.91.99</td>
 97 |             <td>8998</td>
 98 |             <td>高匿</td>
 99 |             <td>HTTP</td>
100 |             <td>0.36 秒</td>
101 |             <td>湖北省武汉市 电信</td>
102 |             <td>17分4秒前</td>
103 |         </tr>
104 | 
105 |         <tr class="warning">
106 |             <td>121.234.234.111</td>
107 |             <td>8998</td>
108 |             <td>高匿</td>
109 |             <td>HTTP</td>
110 |             <td>0.05 秒</td>
111 |             <td>江苏省宿迁市 电信</td>
112 |             <td>25分48秒前</td>
113 |         </tr>
114 | 
115 |         <tr>
116 |             <td>223.198.222.213</td>
117 |             <td>8998</td>
118 |             <td>高匿</td>
119 |             <td>HTTP</td>
120 |             <td>0.35 秒</td>
121 |             <td>海南省三亚市 电信</td>
122 |             <td>14分24秒前</td>
123 |         </tr>
124 | 
125 |         <tr class="warning">
126 |             <td>27.184.55.106</td>
127 |             <td>8998</td>
128 |             <td>高匿</td>
129 |             <td>HTTP</td>
130 |             <td>0.19 秒</td>
131 |             <td>河北省石家庄市 电信</td>
132 |             <td>44分46秒前</td>
133 |         </tr>
134 | 
135 |         <tr>
136 |             <td>123.56.101.126</td>
137 |             <td>8118</td>
138 |             <td>高匿</td>
139 |             <td>HTTP</td>
140 |             <td>1.18 秒</td>
141 |             <td>北京市 阿里云BGP数据中心</td>
142 |             <td>41分56秒前</td>
143 |         </tr>
144 | 
145 |         <tr class="warning">
146 |             <td>36.49.155.8</td>
147 |             <td>8998</td>
148 |             <td>高匿</td>
149 |             <td>HTTP</td>
150 |             <td>0.11 秒</td>
151 |             <td>吉林省吉林市 电信</td>
152 |             <td>29分4秒前</td>
153 |         </tr>
154 | 
155 |         <tr>
156 |             <td>218.0.136.151</td>
157 |             <td>8998</td>
158 |             <td>高匿</td>
159 |             <td>HTTP</td>
160 |             <td>3.16 秒</td>
161 |             <td>浙江省台州市 电信</td>
162 |             <td>16分9秒前</td>
163 |         </tr>
164 | 
165 |         <tr class="warning">
166 |             <td>125.120.8.56</td>
167 |             <td>808</td>
168 |             <td>高匿</td>
169 |             <td>HTTP</td>
170 |             <td>0.07 秒</td>
171 |             <td>浙江省杭州市 (下城区/西湖区)电信</td>
172 |             <td>27分50秒前</td>
173 |         </tr>
174 | 
175 |         <tr>
176 |             <td>182.34.140.155</td>
177 |             <td>8998</td>
178 |             <td>高匿</td>
179 |             <td>HTTP</td>
180 |             <td>0.16 秒</td>
181 |             <td>山东省淄博市 电信</td>
182 |             <td>24分14秒前</td>
183 |         </tr>
184 | 
185 |         <tr class="warning">
186 |             <td>222.242.39.251</td>
187 |             <td>8998</td>
188 |             <td>高匿</td>
189 |             <td>HTTP</td>
190 |             <td>0.10 秒</td>
191 |             <td>湖南省湘西州吉首市 电信</td>
192 |             <td>38分44秒前</td>
193 |         </tr>
194 | 
195 |         </tbody>
196 |     </table>
197 | 
198 |     <div class="page">
199 |         1&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/2.html#ip">2</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/3.html#ip">3</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/4.html#ip">4</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/5.html#ip">5</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/6.html#ip">6</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/7.html#ip">7</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/8.html#ip">8</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/9.html#ip">9</a>&nbsp;&nbsp;&nbsp;<a href="/dailiip/1/10.html#ip">10</a>
200 |     </div>
201 |     <div style="text-align:center;width:100%;margin-top:20px;">
202 |         <a href="/buy.html" class="buy_btn_red">想要更多、质量更好的代理IP，请点这里购买。</a></div>
203 | </div>
204 | 
205 | <div class="online">
206 |     <ul class="online">
207 |         <li class="li"><a href="http://wpa.qq.com/msgrd?v=3&uin=2873763815&site=kxdaili.com&menu=yes" class="ts" target=_blank>2873763815</a><a href="http://wpa.qq.com/msgrd?v=3&uin=2873763815&site=kxdaili.com&menu=yes" class="o_1" target=_blank>qq</a></li>
208 |         <!--<li class="li"><span class="ts">400-</span><a href="#" class="o_2">tel</a></li>-->
209 |         <li class="li"><a href="http://wpa.qq.com/msgrd?v=3&uin=2873763815&site=kxdaili.com&menu=yes" class="ts" target=_blank>意见建议</a><a href="http://wpa.qq.com/msgrd?v=3&uin=2873763815&site=kxdaili.com&menu=yes" class="o_5" target=_blank>qq</a></li>
210 |         <div class="yd-feedback-suggestion">
211 |             <div class="fb-title">
212 |                 <strong>意见反馈</strong>
213 |                 <span class="fb-suggestion-tip">温馨提示：紧急问题请直接联系客服</span>
214 |             </div>
215 | 
216 |             <div class="fb-close">Close</div>
217 | 
218 |             <div class="fb-suggestion-content">
219 |                 <ul class="fb-attitudes">
220 |                     <li value="1" class="fb-attitude-suggest fb-selected">建议</li>
221 |                     <li value="2" class="fb-attitude-criticize">批评</li>
222 |                     <li value="3" class="fb-attitude-praise">表扬</li>
223 |                 </ul>
224 |                 <form name="suggestion">
225 |                     <textarea class="" name="content"></textarea>
226 | 
227 |                     <input type="button" class="fb-input-button" name="submit" id="feed_submit" value="提交">
228 |                     <label class="errortip"></label>
229 |                     <input type="hidden" name="type" value="1">
230 |                 </form>
231 |             </div>
232 |         </div>
233 |         </li>
234 |         <li class="li"><a href="#" class="o_4">top</a></li>
235 |     </ul>
236 | </div>
237 | <!--end在线浮动工具-->
238 | 
239 | 
240 | <script type="text/javascript" src="/public/view/default/js/jquery-1.9.1.min.js"></script>
241 | <script type="text/javascript" src="/public/view/default/js/global.js"></script>
242 | <script type="text/javascript" src="/public/view/default/js/flash.js"></script>
243 | 
244 | <!--[if lt IE 10]>
245 | <script type="text/javascript" src="/public/view/default/js/pie.js"></script>
246 | <![endif]-->
247 | <script language="javascript">
248 | $(function () {
249 | if (window.PIE) {
250 | $('.rounded').each(function () {
251 | PIE.attach(this);
252 | });
253 | }
254 | });
255 | </script>
256 | <dl class="link_th"><dd class="th">友情链接</dd></dl>
257 | <ul class="clear_div h_link th">
258 |     <li>
259 |         <a href="https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=0&rsv_idx=1&tn=baidu&wd=%E5%BC%80%E5%BF%83%E4%BB%A3%E7%90%86IP%E5%B9%B3%E5%8F%B0&rsv_pq=a527cf670000efb9&rsv_t=2bdc76YzwEEZGZGRwERsxX3%2BV5q2wxYefiPSAqpVQCAZ%2FDINTkB2pDYfYUQ&rsv_enter=1&rsv_sug3=23&rsv_sug1=2&rsv_sug2=0&inputT=5873&rsv_sug4=6133" target="_blank">百度-开心代理IP平台</a>
260 |         <a href="http://www.ip181.com/" target="_blank">代理IP检测平台</a>
261 | 
262 |     </li>
263 | </ul>
264 | 
265 | 
266 | <!--end文件脚-->
267 | 
268 | <div class="clear_div2 footer_d"><ul class="clear_div2 footer_d">
269 |     <li>&copy;2015 KXDAILI,INC.上海云海信息科技有限公司，All right reserved.<p>
270 |         客服QQ：2873763815&nbsp;&nbsp;&nbsp;QQ交流群：549703797&nbsp;&nbsp;&nbsp;增值电信业务经营许可证号：沪B2—20140042
271 |     </p></li>
272 | </ul></div>
273 | <!--end文件脚下-->
274 | 
275 | <!--[if IE 6]>
276 | <script type="text/javascript" src="js/DD_belatedPNG.js"></script>
277 | <script>
278 | DD_belatedPNG.fix('.png_bj');
279 | </script>
280 | <![endif]-->
281 | 
282 | <div style="display:none"><script language="javascript" type="text/javascript" src="http://js.users.51.la/17751595.js"></script></div>
283 | </body>
284 | </html>


--------------------------------------------------------------------------------
/crawler/src/test/resources/html/coobobo.html:
--------------------------------------------------------------------------------
  1 | 
  2 | <!DOCTYPE html>
  3 | <html lang="zh">
  4 | <head>
  5 |     <meta charset="utf-8">
  6 |     <meta name="renderer" content="webkit">
  7 |     <title> 酷伯伯HTTP代理</title>
  8 |     <meta name="author" content="http://www.coobobo.com">
  9 |     <meta name=keywords content="酷伯伯代理, 匿名http代理, 高匿http代理, 免费http代理">
 10 |     <meta name="description" content=" 酷伯伯代理,提供 免费http代理, 高匿http代理, 匿名http代理, 普及http代理知识">
 11 | 
 12 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 13 |     <meta name="viewport" content="width=device-width, initial-scale=1">
 14 | 
 15 |     <link rel="icon" type="image/png" href="/assets/img/coobobo.png">
 16 |     <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet">
 17 |     <link href="http://cdn.bootcss.com/font-awesome/4.5.0/css/font-awesome.min.css" rel="stylesheet">
 18 |     <link href="/assets/css/style.css" rel="stylesheet">
 19 | 
 20 |     <meta content="index,follow" name="robots"/>
 21 |     <meta content="index,follow" name="GOOGLEBOT"/>
 22 | 
 23 | 
 24 |     <!--[if lt IE 9]>
 25 |     <script src="http://cdn.bootcss.com/html5shiv/3.7.3/html5shiv.min.js"></script>
 26 |     <script src="http://cdn.bootcss.com/respond.js/1.4.2/respond.min.js"></script>
 27 |     <![endif]-->
 28 | </head>
 29 | <body>
 30 | <header class="navbar navbar-static-top coobobo-nav" id="top" role="banner">
 31 |     <div class="container">
 32 |         <div class="navbar-header">
 33 |             <button class="navbar-toggle collapsed" type="button" data-toggle="collapse" data-target="#bs-navbar"
 34 |                     aria-controls="bs-navbar" aria-expanded="false"><span class=sr-only>Toggle navigation</span> <span
 35 |                     class=icon-bar></span> <span class=icon-bar></span> <span class=icon-bar></span></button>
 36 |             <a href="/" class="navbar-brand"><img alt="Brand" class="img-responsive" src="/assets/img/coobobo.png"/>
 37 |                 酷伯伯HTTP代理</a></div>
 38 |         <nav id=bs-navbar class="collapse navbar-collapse">
 39 |             <ul class="nav navbar-nav">
 40 |                 <li class="active"><a href="/"><i class="fa fa-home" aria-hidden="true"></i> 首页</a></li>
 41 |                 <li class=""><a href="/free-http-proxy">实时免费HTTP代理</a></li>
 42 |                 <li class=""><a href="/free-http-proxy-everyday">每日免费HTTP代理</a></li>
 43 |             </ul>
 44 |             <ul class="nav navbar-nav navbar-right">
 45 |                 <li class="no-background" style="margin-right: 5px;">
 46 |                     <a target="_blank"
 47 |                        href="http://shang.qq.com/wpa/qunwpa?idkey=d3cdd3221debd6571e3f1e52ee23278f2e31fd35b4ee1ed9a7c0f4e773624275"
 48 |                        style="padding: 14px 0;">
 49 |                         <img src="http://pub.idqqimg.com/wpa/images/group.png" alt="酷伯伯交流群" title="酷伯伯交流群"/>
 50 |                     </a>
 51 |                 </li>
 52 |                 <li class="no-background action">
 53 |                     <a href="/user/register">注册</a>
 54 |                 </li>
 55 |                 <li class="no-background action">
 56 |                     <a href="/user/login">登录</a>
 57 |                 </li>
 58 | 
 59 |             </ul>
 60 |         </nav>
 61 |     </div>
 62 | </header>
 63 | <main class="coobobo-masthead" id="content" role="main" tabindex="-1">
 64 |     <div class="container">
 65 |         <p class="strong">现在可以 免费获取 酷伯伯HTTP代理 VIP 啦: <a href="http://coobobodaili.sxl.cn/" target="_blank">猛击这里!</a></p>
 66 |     </div>
 67 | </main>
 68 | <div class="container">
 69 |     <div class="row">
 70 |         <div class="col-md-12">
 71 | 
 72 |             <h1>免费HTTP代理</h1>
 73 | 
 74 | 
 75 |             <table class="table table-bordered table-hover table-condensed">
 76 |                 <thead>
 77 |                 <tr>
 78 | 
 79 |                     <th>IP</th>
 80 |                     <th>端口</th>
 81 |                     <th>匿名度</th>
 82 |                     <th>位置</th>
 83 |                     <th>响应时间</th>
 84 |                     <th>最后验证时间</th>
 85 |                 </tr>
 86 |                 </thead>
 87 |                 <tbody>
 88 |                 <tr>
 89 |                     <td>80.88.90.181</td>
 90 |                     <td>8080</td>
 91 |                     <td>透明</td>
 92 |                     <td>
 93 |                         意大利
 94 |                     </td>
 95 |                     <td>
 96 |                         <div class="progress" style="margin-bottom: 3px;">
 97 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
 98 |                                  style="min-width: 3em; width: 66.61%;">
 99 |                                 6.7秒
100 |                             </div>
101 |                         </div>
102 | 
103 |                     </td>
104 |                     <td>
105 |                 <span data-toggle="from-now">
106 |                     2016-12-15 13:19:30
107 |                 </span>
108 |                     </td>
109 |                 </tr>
110 |                 <tr>
111 |                     <td>138.121.52.146</td>
112 |                     <td>8080</td>
113 |                     <td>透明</td>
114 |                     <td>
115 |                         巴西
116 |                     </td>
117 |                     <td>
118 |                         <div class="progress" style="margin-bottom: 3px;">
119 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
120 |                                  style="min-width: 3em; width: 100.0%;">
121 |                                 10.1秒
122 |                             </div>
123 |                         </div>
124 | 
125 |                     </td>
126 |                     <td>
127 |                 <span data-toggle="from-now">
128 |                     2016-12-15 13:19:27
129 |                 </span>
130 |                     </td>
131 |                 </tr>
132 |                 <tr>
133 |                     <td>36.66.215.93</td>
134 |                     <td>8080</td>
135 |                     <td>透明</td>
136 |                     <td>
137 |                         印度尼西亚
138 |                     </td>
139 |                     <td>
140 |                         <div class="progress" style="margin-bottom: 3px;">
141 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
142 |                                  style="min-width: 3em; width: 100.0%;">
143 |                                 13.5秒
144 |                             </div>
145 |                         </div>
146 | 
147 |                     </td>
148 |                     <td>
149 |                 <span data-toggle="from-now">
150 |                     2016-12-15 13:18:45
151 |                 </span>
152 |                     </td>
153 |                 </tr>
154 |                 <tr>
155 |                     <td>41.160.223.252</td>
156 |                     <td>8080</td>
157 |                     <td>透明</td>
158 |                     <td>
159 |                         南非
160 |                     </td>
161 |                     <td>
162 |                         <div class="progress" style="margin-bottom: 3px;">
163 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
164 |                                  style="min-width: 3em; width: 100.0%;">
165 |                                 12.1秒
166 |                             </div>
167 |                         </div>
168 | 
169 |                     </td>
170 |                     <td>
171 |                 <span data-toggle="from-now">
172 |                     2016-12-15 13:18:44
173 |                 </span>
174 |                     </td>
175 |                 </tr>
176 |                 <tr>
177 |                     <td>192.99.43.195</td>
178 |                     <td>3128</td>
179 |                     <td>透明</td>
180 |                     <td>
181 |                     </td>
182 |                     <td>
183 |                         <div class="progress" style="margin-bottom: 3px;">
184 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
185 |                                  style="min-width: 3em; width: 94.09%;">
186 |                                 9.4秒
187 |                             </div>
188 |                         </div>
189 | 
190 |                     </td>
191 |                     <td>
192 |                 <span data-toggle="from-now">
193 |                     2016-12-15 13:18:41
194 |                 </span>
195 |                     </td>
196 |                 </tr>
197 |                 <tr>
198 |                     <td>159.203.129.234</td>
199 |                     <td>3128</td>
200 |                     <td>透明</td>
201 |                     <td>
202 |                         美国
203 |                     </td>
204 |                     <td>
205 |                         <div class="progress" style="margin-bottom: 3px;">
206 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
207 |                                  style="min-width: 3em; width: 80.41%;">
208 |                                 8.0秒
209 |                             </div>
210 |                         </div>
211 | 
212 |                     </td>
213 |                     <td>
214 |                 <span data-toggle="from-now">
215 |                     2016-12-15 13:18:40
216 |                 </span>
217 |                     </td>
218 |                 </tr>
219 |                 <tr>
220 |                     <td>139.59.249.168</td>
221 |                     <td>8080</td>
222 |                     <td>透明</td>
223 |                     <td>
224 |                         新加坡
225 |                     </td>
226 |                     <td>
227 |                         <div class="progress" style="margin-bottom: 3px;">
228 |                             <div class="progress-bar progress-bar-warning" role="progressbar"
229 |                                  style="min-width: 3em; width: 59.86%;">
230 |                                 6.0秒
231 |                             </div>
232 |                         </div>
233 | 
234 |                     </td>
235 |                     <td>
236 |                 <span data-toggle="from-now">
237 |                     2016-12-15 13:18:38
238 |                 </span>
239 |                     </td>
240 |                 </tr>
241 |                 <tr>
242 |                     <td>115.127.91.18</td>
243 |                     <td>8080</td>
244 |                     <td>透明</td>
245 |                     <td>
246 |                         孟加拉国
247 |                     </td>
248 |                     <td>
249 |                         <div class="progress" style="margin-bottom: 3px;">
250 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
251 |                                  style="min-width: 3em; width: 90.23%;">
252 |                                 9.0秒
253 |                             </div>
254 |                         </div>
255 | 
256 |                     </td>
257 |                     <td>
258 |                 <span data-toggle="from-now">
259 |                     2016-12-15 13:18:37
260 |                 </span>
261 |                     </td>
262 |                 </tr>
263 |                 <tr>
264 |                     <td>62.220.55.27</td>
265 |                     <td>8080</td>
266 |                     <td>透明</td>
267 |                     <td>
268 |                         俄罗斯
269 |                     </td>
270 |                     <td>
271 |                         <div class="progress" style="margin-bottom: 3px;">
272 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
273 |                                  style="min-width: 3em; width: 76.53%;">
274 |                                 7.7秒
275 |                             </div>
276 |                         </div>
277 | 
278 |                     </td>
279 |                     <td>
280 |                 <span data-toggle="from-now">
281 |                     2016-12-15 13:18:35
282 |                 </span>
283 |                     </td>
284 |                 </tr>
285 |                 <tr>
286 |                     <td>217.61.3.240</td>
287 |                     <td>3128</td>
288 |                     <td>透明</td>
289 |                     <td>
290 |                         丹麦
291 |                     </td>
292 |                     <td>
293 |                         <div class="progress" style="margin-bottom: 3px;">
294 |                             <div class="progress-bar progress-bar-danger" role="progressbar"
295 |                                  style="min-width: 3em; width: 100.0%;">
296 |                                 10.1秒
297 |                             </div>
298 |                         </div>
299 | 
300 |                     </td>
301 |                     <td>
302 |                 <span data-toggle="from-now">
303 |                     2016-12-15 13:18:35
304 |                 </span>
305 |                     </td>
306 |                 </tr>
307 |                 <tr>
308 |                     <td>36.249.235.124</td>
309 |                     <td>8998</td>
310 |                     <td>匿名</td>
311 |                     <td>
312 |                         中国
313 |                         漳州市
314 |                     </td>
315 |                     <td>
316 |                         <div class="progress" style="margin-bottom: 3px;">
317 |                             <div class="progress-bar progress-bar-success" role="progressbar"
318 |                                  style="min-width: 3em; width: 3.3%;">
319 |                                 0.3秒
320 |                             </div>
321 |                         </div>
322 | 
323 |                     </td>
324 |                     <td>
325 |                 <span data-toggle="from-now">
326 |                     2016-12-15 13:18:31
327 |                 </span>
328 |                     </td>
329 |                 </tr>
330 |                 <tr>
331 |                     <td>27.208.194.153</td>
332 |                     <td>8998</td>
333 |                     <td>匿名</td>
334 |                     <td>
335 |                         中国
336 |                         威海市
337 |                     </td>
338 |                     <td>
339 |                         <div class="progress" style="margin-bottom: 3px;">
340 |                             <div class="progress-bar progress-bar-success" role="progressbar"
341 |                                  style="min-width: 3em; width: 2.26%;">
342 |                                 0.2秒
343 |                             </div>
344 |                         </div>
345 | 
346 |                     </td>
347 |                     <td>
348 |                 <span data-toggle="from-now">
349 |                     2016-12-15 13:18:31
350 |                 </span>
351 |                     </td>
352 |                 </tr>
353 |                 <tr>
354 |                     <td>114.237.79.101</td>
355 |                     <td>8998</td>
356 |                     <td>匿名</td>
357 |                     <td>
358 |                         中国
359 |                         连云港市
360 |                     </td>
361 |                     <td>
362 |                         <div class="progress" style="margin-bottom: 3px;">
363 |                             <div class="progress-bar progress-bar-success" role="progressbar"
364 |                                  style="min-width: 3em; width: 3.47%;">
365 |                                 0.3秒
366 |                             </div>
367 |                         </div>
368 | 
369 |                     </td>
370 |                     <td>
371 |                 <span data-toggle="from-now">
372 |                     2016-12-15 13:18:30
373 |                 </span>
374 |                     </td>
375 |                 </tr>
376 |                 <tr>
377 |                     <td>122.141.5.140</td>
378 |                     <td>8998</td>
379 |                     <td>匿名</td>
380 |                     <td>
381 |                         中国
382 |                         松原市
383 |                     </td>
384 |                     <td>
385 |                         <div class="progress" style="margin-bottom: 3px;">
386 |                             <div class="progress-bar progress-bar-success" role="progressbar"
387 |                                  style="min-width: 3em; width: 5.7%;">
388 |                                 0.6秒
389 |                             </div>
390 |                         </div>
391 | 
392 |                     </td>
393 |                     <td>
394 |                 <span data-toggle="from-now">
395 |                     2016-12-15 13:18:30
396 |                 </span>
397 |                     </td>
398 |                 </tr>
399 |                 <tr>
400 |                     <td>202.47.236.251</td>
401 |                     <td>80</td>
402 |                     <td>匿名</td>
403 |                     <td>
404 |                         泰国
405 |                     </td>
406 |                     <td>
407 |                         <div class="progress" style="margin-bottom: 3px;">
408 |                             <div class="progress-bar progress-bar-success" role="progressbar"
409 |                                  style="min-width: 3em; width: 9.38%;">
410 |                                 0.9秒
411 |                             </div>
412 |                         </div>
413 | 
414 |                     </td>
415 |                     <td>
416 |                 <span data-toggle="from-now">
417 |                     2016-12-15 13:18:29
418 |                 </span>
419 |                     </td>
420 |                 </tr>
421 | 
422 | 
423 |                 </tbody>
424 |             </table>
425 | 
426 |             <nav class="text-center">
427 |                 <ul class="pagination">
428 |                     <li>
429 |                         <a href="/free-http-proxy/0" aria-label="Previous">
430 |                             <span aria-hidden="true">&laquo;</span>
431 |                         </a>
432 |                     </li>
433 |                     <li class="active"><a href="#">1</a></li>
434 | 
435 |                     <li><a href="/free-http-proxy/2">2</a></li>
436 | 
437 |                     <li><a href="/free-http-proxy/3">3</a></li>
438 | 
439 |                     <li><a href="/free-http-proxy/4">4</a></li>
440 | 
441 |                     <li><a href="/free-http-proxy/5">5</a></li>
442 | 
443 |                     <li><a href="/free-http-proxy/6">6</a></li>
444 | 
445 |                     <li><a href="/free-http-proxy/7">7</a></li>
446 | 
447 |                     <li><a href="/free-http-proxy/8">8</a></li>
448 | 
449 |                     <li><a href="/free-http-proxy/9">9</a></li>
450 | 
451 |                     <li><a href="/free-http-proxy/10">10</a></li>
452 | 
453 |                     <li>
454 |                         <a href="/free-http-proxy/2" aria-label="Next">
455 |                             <span aria-hidden="true">&raquo;</span>
456 |                         </a>
457 |                     </li>
458 |                 </ul>
459 |             </nav>
460 |         </div>
461 |     </div>
462 | </div>
463 | 
464 | <footer class=bs-docs-footer role=contentinfo>
465 |     <div class=container>
466 |         <ul class=bs-docs-footer-links>
467 |             <li><a href="/">酷伯伯HTTP代理</a></li>
468 |             <li><a href="/free-http-proxy">免费HTTP代理</a></li>
469 |             <li><a href="/free-http-proxy-everyday">每日HTTP代理</a></li>
470 |             <li>
471 |                 <a target="_blank"
472 |                    href="http://shang.qq.com/wpa/qunwpa?idkey=d3cdd3221debd6571e3f1e52ee23278f2e31fd35b4ee1ed9a7c0f4e773624275"
473 |                    style="padding: 14px 0;">
474 |                     <img src="http://pub.idqqimg.com/wpa/images/group.png" alt="酷伯伯交流群" title="酷伯伯交流群"/>
475 |                 </a>
476 |             </li>
477 |         </ul>
478 |     </div>
479 | </footer>
480 | 
481 | 
482 | 
483 | <script>
484 |     (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
485 |                 (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
486 |             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
487 |     })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
488 | 
489 |     ga('create', 'UA-2770908-13', 'auto');
490 |     ga('send', 'pageview');
491 | </script>
492 | 
493 | <script>
494 |     (function(){
495 |         var bp = document.createElement('script');
496 |         var curProtocol = window.location.protocol.split(':')[0];
497 |         if (curProtocol === 'https') {
498 |             bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
499 |         }
500 |         else {
501 |             bp.src = 'http://push.zhanzhang.baidu.com/push.js';
502 |         }
503 |         var s = document.getElementsByTagName("script")[0];
504 |         s.parentNode.insertBefore(bp, s);
505 |     })();
506 | </script>
507 | 
508 | 
509 | <script src="http://cdn.bootcss.com/jquery/1.12.1/jquery.min.js"></script>
510 | <script src="http://cdn.bootcss.com/bootstrap/3.3.6/js/bootstrap.min.js"></script>
511 | <script src="http://cdn.bootcss.com/moment.js/2.11.2/moment.min.js"></script>
512 | <script src="http://cdn.bootcss.com/moment.js/2.11.2/locale/zh-cn.js"></script>
513 | <script src=/assets/js/ie10.js></script>
514 | <script src=/assets/js/app.js></script>
515 | <script type="text/javascript">var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_1259226546'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s4.cnzz.com/stat.php%3Fid%3D1259226546%26show%3Dpic' type='text/javascript'%3E%3C/script%3E"));</script>
516 | <script>
517 |     $(function(){
518 |         $("a[href^='http://www.cnzz.com/stat/website.php?web_id=']").hide();
519 |     });
520 | </script>
521 | </body>
522 | </html>


--------------------------------------------------------------------------------
/crawler/src/test/resources/html/kuaidaili.html:
--------------------------------------------------------------------------------
  1 | 
  2 | <!doctype html>
  3 | <html>
  4 | <head>
  5 |     <meta charset="utf-8">
  6 |     <meta name="viewport" content="width=device-width, initial-scale=1,user-scalable=no;">
  7 |     <title>快代理 - 高速http代理ip每天更新</title>
  8 | 
  9 |     <meta name="keywords" content="快代理,http代理,https代理,私密代理,代理服务器,高速http代理,代理,代理ip,私密代理ip,爬虫代理ip,刷单代理ip,随机五位端口,独享代理ip,稳定代理ip,代理ip地址,花刺代理ip,ip代理,ip代理软件,免费代理服务器,免费代理ip,高匿代理,高匿代理ip,最新代理ip,代理api接口,代理服务器ip,代理地址,代理列表,代理ip购买" />
 10 |     <meta name="description" content="快代理专业为您提供代理ip购买|代理服务器|随机五位端口|高匿代理|高匿代理ip|爬虫代理ip|刷单代理ip|私密代理ip|独享代理|独享代理ip|高速http代理|免费代理ip|代理ip地址|最新免费代理ip每天更新，想要了解更多代理服务|代理地址|代理列表的详情，就到快代理。" />
 11 |     <meta content="index,follow" name="robots"/>
 12 |     <meta content="index,follow" name="GOOGLEBOT"/>
 13 |     <meta content="快代理"  name="Author"/>
 14 |     <meta name="renderer" content="webkit">
 15 |     <meta name="baidu_union_verify" content="c087a423b52225f404d4c97e59e53464">
 16 |     <meta name="google-site-verification" content="Pd8y4Id4xJSxMvj-OwUhaZaK7COpr-8LcANUG30jxW8" />
 17 | 
 18 |     <link rel="shortcut icon" href="http://img.kuaidaili.com/img/favicon.ico?v=3" type="image/x-icon">
 19 | 
 20 | 
 21 |     <style>
 22 | body{font-family:"微软雅黑",Helvetica;font-size:13px;line-height:160%;color:#666;background-color:#fff;-webkit-tap-highlight-color:rgba(0,0,0,0);margin:0 auto;padding:20px 10px}a img{border:0}.placeholder{color:#aaa}form,table,td,h1,h2,h3,h4,h5,ul,ol,li,p{margin:0;padding:0;border:0;list-style:none}#header{position:relative;max-width:1100px;min-height:73px;margin:0 auto;border-bottom:1px solid #ddd;clear:right}#logo{width:136px;position:absolute;bottom:0}#logo img{width:136px;height:44px}.header_top{float:right;padding:0;line-height:30px}.header_top .login{display:inline-block;margin-right:25px}.header_top .login .log{color:#b94a48}.header_top a{text-decoration:none;color:#49afcd}.header_top a:hover{text-decoration:underline}.header_top a img{vertical-align:-22%}.header_top .splt{color:#bbb;margin:0 8px}.header_top .btn{background-color:#49afcd;padding:3px 5px 3px 5px;color:#fff;font-weight:bold;border-radius:3px;transition:background-color .2s linear}.header_top .btn:hover{background-color:#3a87ad;text-decoration:none}.header_top .faq_btn{background-color:#aaa}.header_top .faq_btn:hover{background-color:#1a1a1a}.header_top .event{margin-right:50px}.header_top .event a{line-height:30px;color:red;text-decoration:none}.header_top .event a:hover{text-decoration:underline}.header_top .event a img{vertical-align:sub}@media screen and (min-width:960px){.header_top{min-width:710px}.header_top .wrap{float:right}#nav ul li.active .upper{margin-left:11px}}#nav ul{margin-top:10px;float:right}#nav>ul li{float:left;display:inline-block;height:37px;margin-right:10px}#nav>ul li:last-child{margin-right:0}#nav>ul li a{display:inline-block;color:#979795;font-size:16px;font-weight:bold;line-height:32px;vertical-align:middle;padding:0 10px;text-decoration:none;border-bottom:2px solid #c8e6e0}#nav>ul li a:hover{border-bottom:2px solid #49afcd;color:#333}#nav>ul>li.active>a{background-color:#49afcd;color:#fff;border-bottom:2px solid #49afcd}#nav>ul li .upper{position:absolute;margin-top:-5px}#container{max-width:1100px;min-height:500px;margin:0 auto;padding-top:18px}.taglineWrap{background:#eee;min-height:20px;margin:0 -10px}.taglineContent{max-width:1100px;margin:0 auto;padding:24px 19px;word-wrap:break-word}.taglineContent div{display:inline-block}.taglineContent h1{margin-bottom:20px}.taglineContent h2{margin-bottom:15px}.taglineContent span{display:inline-block;width:110px}.taglineContent li{font-size:14px;margin:6px 0}.taglineContent p{font-size:14px;margin:3px 0}#footer{margin-top:30px}#footer span{color:#bbb}#footer .footnav{margin-bottom:5px}.footnav a{padding:0 6px;margin:0 5px;color:#999;text-decoration:none;display:inline-block}.footnav a:hover{color:#fff;background:#49afcd;line-height:23px;text-decoration:none;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.footnav span{color:#ddd}#footer2{margin:30px 0 20px 0;padding:30px 0;border-top:1px solid #efefef}#footer2 .container{width:1100px;margin:0 auto}#footer2 .copyright{width:320px;color:#bbb;float:left;display:inline-block;line-height:40px;vertical-align:top}#footer2 .footnav{display:inline-block;line-height:40px;vertical-align:top;float:left}#footer2 .icon{display:inline-block;float:right;line-height:40px;vertical-align:top}ul.onMShow{width:77px;position:fixed;bottom:93px;right:0;z-index:100}ul.onMShow li{list-style:none;background-position:left top;background-repeat:no-repeat}ul.onMShow li a{position:relative;display:block;width:70px;height:50px;border:0;opacity:.8}ul.onMShow li a div{display:none}ul.onMShow li a:hover{background::none;display:block;width:auto}ul.onMShow li a:hover div{display:block;position:absolute;padding:0 13px 0 10px;line-height:43px;width:auto;right:87px;top:0;background-color:#49afcd;color:#fff;cursor:pointer;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}ul.onMShow li a:hover div.two{line-height:21px}span.bt1,span.bt2,span.bt3{width:43px;height:43px;display:block;padding:0;position:absolute;left:0;top:0;background:#49afcd url(../img/licon.png) 0 0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}span.bt2{background-position:0 -40px}span.bt3{background-position:0 -80px}span.bt1:hover,span.bt2:hover,span.bt3:hover{opacity:1}.trans a{color:#fcfcfc}h1{color:#2f2f2f;font-size:24px}h2{color:#2f2f2f;font-size:20px}h3{color:#2f2f2f;font-size:18px}h4{color:#2f2f2f;font-size:16px}h5{color:#2f2f2f;font-size:14px}.c_title{margin:10px 0 30px 0}.c_title h2{display:inline}.center{width:100%;text-align:center}.right{float:right}.contact form .control-group:after{content:".";display:block;height:0;visibility:hidden;clear:both}.control-group{margin-bottom:15px;clear:both}.control-group .price{color:#F60;font-size:20px;font-weight:bold;margin-right:1px}.contact form .control-label{float:left;color:#666;text-align:left;width:140px;font-size:16px;margin:5px 0}.control-group .control-label.active{color:#b94a48;font-weight:bold}.control-group .controls .hint{font-size:13px}.contact form .controls{float:left;margin-left:0}#listnav{margin:20px 0 30px 0}#listnav ul{text-align:center;border:solid;border-width:0;padding:2px 0 0 0}#listnav li{display:inline;padding:0 0 0 10px;list-style:none;border:1px}#listnav a{color:#9d9d9d;text-decoration:none;padding:4px 6px;border:1px solid #dfdfdf}#listnav a:hover{color:#9d9d9d;background:#e5f3f6;border:1px solid #49afcd}#listnav a.active{color:#fff;background:#49afcd;border:1px solid #49afcd}#list table{margin-left:auto;margin-right:auto}#list p{margin:0 0 10px 0}.hint{color:#aaa}.control-label .star{display:inline-block;height:10px;font-size:26px;color:red;vertical-align:top;padding-top:5px;margin-left:2px}span.hint{margin-left:10px}.controls b{margin-right:20px}.controls .err{color:red;font-weight:bold}.controls a,.submit a{color:#49afcd;text-decoration:none}.controls a:hover,a:hover.hint{color:#2f96b4;text-decoration:underline}.btn a{height:30px;line-height:30px;padding:0 11px;font-size:16px;color:white;display:inline-block;text-decoration:none;background:#49afcd}.btn a:hover{background:#2f96b4;text-decoration:none}.btn a.disabled{background:#a3aeb1}.btn.center>a{margin-bottom:20px}.taglineWrap .btn.center>a{margin-bottom:0}.invisible{visibility:hidden;position:absolute}.fillhint{margin-left:140px}.submit{margin-left:140px;margin-top:10px;clear:both}.buy{float:right}.buy a{color:#49afcd;font-size:13px;font-weight:bold;text-decoration:none}.buy a:hover{text-decoration:underline}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #eee;border:1px solid rgba(0,0,0,0.05);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;word-wrap:break-word}.well h2,.well h3{margin-bottom:10px}.well b{display:inline-block;width:110px;font-weight:normal;line-height:2;vertical-align:top}.well b a.active{font-size:14px;color:#b94a48;text-decoration:none}.well h3 a,.well b a{font-size:13px;color:#49afcd;text-decoration:none}.well h3.active{color:#b94a48}.well .hint{font-size:13px;float:right;color:#468847;font-weight:normal}.tag{margin-left:3px;font-weight:normal;font-size:9px}.upper{vertical-align:super;color:red}.badge{padding:1px 5px 1px;white-space:nowrap;color:#fff;background-color:#999;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px;margin-left:5px}.badge-sup1{position:relative;top:-10px;margin-left:-5px}.badge-important{background-color:#b94a48}code{padding:2px 4px;margin-left:2px;margin-right:2px;background-color:#f7f7f9;border:1px solid #e1e1e8;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}table{max-width:100%;border-collapse:collapse;border-spacing:0;background-color:transparent}td a{color:#49afcd;text-decoration:none}td a:hover{text-decoration:underline}td .price{color:#F60}td .active{color:#3C0}td .warn{color:#b94a48}td .seq{margin:0 3px;color:#ddd}.table{width:100%;margin-bottom:10px}.table th,.table td{padding:8px;line-height:18px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th.center,.table td.center{text-align:center;width:auto}.table thead th{vertical-align:bottom;font-weight:bold}.table tbody+tbody{border-top:2px solid #ddd}.table-bordered{border:1px solid #ddd;border-left:0;border-collapse:separate;*border-collapse:collapsed;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-noborder{border:1px solid #ddd;border-top:0;border-collapse:separate;*border-collapse:collapsed;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-radius:4px 0 0 0;-moz-border-radius:4px 0 0 0;border-radius:4px 0 0 0}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-radius:0 4px 0 0;-moz-border-radius:0 4px 0 0;border-radius:0 4px 0 0}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;border-radius:0 0 0 4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child{-webkit-border-radius:0 0 4px 0;-moz-border-radius:0 0 4px 0;border-radius:0 0 4px 0}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.odd{background-color:#f9f9f9}.table tbody tr:hover td,.table tbody tr:hover th{background-color:#f5f5f5}.stat span{font-size:16px;color:#2f2f2f}.stat strong{font-size:36px;color:#49afcd;font-weight:bold}.stat .hint{font-size:13px;color:#aaa}.stat_num{color:#49afcd;font-size:24px;font-weight:bold}#uc_content{margin-top:10px}a.submit-a-request i{font-style:normal}a.submit-a-request{color:#49afcd}a.submit-a-request:hover{color:#fff;text-decoration:none}.submit-a-request{border:2px solid #49afcd;border-radius:3px;display:inline;font-weight:normal;line-height:1.2;padding:4px 5px 4px 3px;text-align:left;text-decoration:none}.submit-a-request:hover{background-color:#49afcd;color:#fff;text-decoration:none}.row-fluid{width:100%;overflow:hidden}.row-fluid>[class*="span"]{float:left;margin-left:1.7%}.row-fluid>[class*="span"]:first-child{margin-left:0}.row-fluid>.span3{width:23.604255317%}.row-fluid>.span4{width:27%}.row-fluid>.span9{width:74.30939226%}.sidebar .well{margin-bottom:30px}div.sidebar{display:block}.nav{margin-left:0;list-style:none}.nav li a{text-decoration:none;display:block}.nav li a:hover{text-decoration:none;background-color:#eee}.nav .nav-center{display:block;padding:3px 15px;margin-bottom:7px;font-size:13px;font-weight:bold;line-height:18px;color:#999;text-align:center}.nav .nav-header{display:block;padding:3px 15px;font-size:13px;font-weight:bold;line-height:18px;color:#999;text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-left:15px;padding-right:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-left:-15px;margin-right:-15px}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{height:1px;margin:8px 1px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff;*width:100%;*margin:-5px 0 5px}.nav-list>.active>a,.nav-list>.active>a:hover{background-color:#49afcd}.nav-list i{opacity:.6}.nav-list>.active i{opacity:1}.nav-list a{color:#2f2f2f}.tab-content{display:table;width:100%}.tabs-below .nav-tabs,.tabs-right .nav-tabs,.tabs-left .nav-tabs{border-bottom:0}.tab-content .active,.pill-content .active{display:block}.tab-content .tab-pane,.pill-content .pill-pane{padding:20px 0 0 10px;border-left:1px solid #ddd;border-right:1px solid #ddd;border-bottom:1px solid #ddd;border-radius:0 0 4px 4px}.btt{position:fixed;bottom:45px;right:30px;opacity:.4;background-color:#49afcd;padding:15px;visibility:visible;-webkit-transition:opacity .4s linear;-moz-transition:opacity .4s linear;-ms-transition:opacity .4s linear;-o-transition:opacity .4s linear;transition:opacity .4s linear}.btt:hover{opacity:.8}.bottom_kw{color:#fff}.bottom_kw a{text-decoration:none;color:#fff}.dropdown{position:relative}.caret{display:inline-block;width:0;height:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000;opacity:.3;filter:alpha(opacity=30);content:""}.dropdown .caret{margin-top:14px;margin-left:4px}.dropdown:hover .caret,.open.dropdown .caret{opacity:1;filter:alpha(opacity=100)}.dropdown.active .caret{border-top-color:#fff;opacity:1}.dropmenu{position:absolute;top:100%;left:0;z-index:10;float:left;display:none;min-width:160px;padding:4px 0;margin:0;list-style:none;background-color:#fff;border-color:#ccc;border-color:rgba(0,0,0,0.2);border-style:solid;border-width:1px;-webkit-border-radius:0 0 5px 5px;-moz-border-radius:0 0 5px 5px;border-radius:0 0 5px 5px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box;display:block !important;visibility:hidden;opacity:0;transition:opacity .2s linear}#nav #menu>li.dropdown:hover>.dropmenu{display:block;opacity:1 !important;visibility:visible}#nav .dropmenu{border-radius:0;-moz-border-radius:0;-webkit-border-radius:0;border-color:#e5e5e5;border:0;box-shadow:0 3px 5px rgba(0,0,0,0.5);-webkit-box-shadow:0 3px 5px rgba(0,0,0,0.5);background:#393939;margin-top:-2px}#nav .dropmenu li{display:block;width:100%;height:28px;line-height:28px}#nav .dropmenu a{display:block;width:100%;height:28px;padding:0;text-indent:10px;font-size:14px;line-height:28px;border:0;clear:both;font-weight:normal;color:#fff;white-space:nowrap}#nav .dropmenu li>a:hover,#nav .dropmenu .active>a,#nav .dropmenu .active>a:hover{display:block;width:100%;color:#fff;text-decoration:none;background-color:#49afcd;border:0}.lead{font-size:20px;max-width:700px;margin:0 auto;padding:0 10px;line-height:1.3em}.text-center{text-align:center}@media screen and (max-width:770px){table,tr,td{border-collapse:collapse}.table,.table td,.table tr,.table thead,.table tbody,.table th{display:block;position:relative}.table thead{display:none;position:absolute;top:-9999px;left:-9999px}.table td{position:relative;padding-left:40%}.table td:before{position:absolute;top:6px;left:6px;width:40%;padding-right:10px;white-space:nowrap;content:attr(data-title);font-weight:bold}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#eee}.table th.center,.table td.center{text-align:left;width:auto}#menu{width:100%;display:block}body{padding-top:0}#nav{display:block;overflow:hidden;width:100%}#nav ul{float:left;margin-top:10px}#nav ul li{margin:0;padding:0;font-size:0;width:100%;position:relative;height:auto;margin-bottom:2px}#nav ul li a{background-color:#c1c1bf;color:#fff;border-bottom:0;display:block;height:30px;line-height:30px;text-align:center;font-size:14px;margin:0;padding:0;border-right:1px solid #fff;text-align:left;text-indent:1em;font-weight:normal}#nav ul li.active a{border-bottom:0}#nav ul li a:hover{border-bottom:0;background-color:#9e9e9c;color:#fff}#nav ul li .upper{margin-left:-7px}#nav .dropmenu{padding:0;margin-top:-1px}#nav .dropdown{height:auto;overflow:hidden}#nav #menu>li.dropdown:hover>.dropmenu{background:#393939}#nav .dropmenu{position:relative;width:100%;opacity:1 !important;visibility:visible;background:#393939}#nav .dropmenu li{background:#393939}#nav .dropmenu li:last-child{margin:0}#nav .dropmenu a{display:block;width:100%;height:28px;padding:0;text-indent:20px;font-size:13px;line-height:28px;border:0;clear:both;font-weight:normal;color:#fff;white-space:nowrap}#nav ul.dropmenu li a{background:#393939}#nav .dropmenu li>a:hover,#nav .dropmenu .active>a,#nav .dropmenu .active>a:hover{display:block;text-indent:20px;font-size:13px;width:100%;color:#fff;text-decoration:none;background-color:#08c;border:0}#logo{width:100px;top:20px;bottom:auto}#logo img{width:123px;height:40px}.header_top .login{margin-right:5px}.header_top{max-width:500px;text-align:right}.header_top .wrap{display:block}#welcome{display:none}.header_top .event a{font-size:10px}.header_top .event{margin-right:0}#footer2{margin:20px 0;padding:10px}#footer2 .container{width:100%}#footer2 .copyright{text-align:center;width:100%}#footer2 .icon{text-align:center;display:block;float:none}#footer2 .footnav{text-align:center;padding-bottom:20px;float:none;width:100%}.tag_area{display:-webkit-box;margin-top:20px;position:relative}.tag_area a.label{display:block;-webkit-box-flex:1;text-align:center;padding:10px 0;font-size:16px;width:100%}.tag_area span.buy{position:absolute;display:block;left:0;width:100%;text-align:left;bottom:0}.contact .control-group{margin:20px 0 5px 0}input{margin:0;padding:0}input.span4,input.span3,input.span7,textarea.span7,textarea.span4,.uneditable-input.span4{width:100%;-webkit-box-sizing:border-box;margin:0;padding:4px}.stat strong{font-size:20px}.stat .right{float:none;display:block}}@media screen and (max-width:480px){.buy{display:none}.table{border-top:0}.table td{font-size:12px}.stat strong{font-size:20px}.stat .right{float:none;display:block}#container{padding:0}ul.onMShow{width:53px}.btt{right:6px}.header_top{max-width:230px;text-align:right}.header_top .event{margin-right:0;margin-left:5px}#listnav{overflow:hidden;padding-top:10px;margin:5px 0 10px 0}#listnav ul{display:block}#listnav li{padding:0 4px;margin-bottom:10px;float:left}#listnav a{padding:3px 15px;display:inline}#footer2 .beian { color: #bbb; text-decoration: none; } #footer2 .beian:hover { color: #49afcd; }}
 23 | </style>
 24 | 
 25 | 
 26 |     <meta name="renderer" content="webkit">
 27 |     <meta name="baidu-site-verification" content="AO3Q6dKj9R" />
 28 |     <meta name="sogou_site_verification" content="9ELczs5cQc"/>
 29 |     <meta name="360-site-verification" content="feeadcad97dd7093f9abb1ee5285f031" />
 30 | </head>
 31 | 
 32 | <body>
 33 | 
 34 | <div id="header">
 35 |     <div id="logo"><a href="/"><img src="http://img.kuaidaili.com/img/kdl_logo.jpg?v=1" alt="快代理logo"/></a></div>
 36 |     <div class="header_top">
 37 | 
 38 |         <span class="event">
 39 | 
 40 |         </span>
 41 | 
 42 |         <span class="wrap">
 43 |             <span id="welcome"></span>
 44 |             <span class="login">
 45 |                 <span class="splt"></span><a class="uc_btn btn" href="/usercenter/"><i class="icon-user icon-white"></i>&nbsp;会员中心</a><span id="noti"></span>
 46 |             </span>
 47 | 
 48 |             <a target="_blank" href="http://wpa.qq.com/msgrd?v=3&amp;uin=2162481273&amp;site=qq&amp;menu=yes"><img border="0" width="77" height="22" src="http://img.kuaidaili.com/img/button_21.gif" alt="留言咨询" title="留言咨询"/></a>
 49 |         </span>
 50 |     </div>
 51 |     <div id="nav">
 52 |         <ul id="menu">
 53 |             <li id="menu_list"><a href="/">首页</a></li>
 54 |             <li id="menu_free"><a href="/free/">免费代理</a></li>
 55 |             <li id="menu_pricing"><a href="/pricing/">购买代理</a></li>
 56 |             <li id="menu_ops"><a href="/dist/">开放代理</a></li>
 57 | 
 58 |             <li id="menu_kps"><a href="/kps/">独享代理</a></li>
 59 |             <li id="menu_fetch" class="dropdown"><a href="javascript:void(0);">代理提取<b class="caret"></b></a>
 60 |                 <ul class="dropmenu">
 61 |                     <li><a href="/fetch/">提取开放代理</a></li>
 62 |                     <li><a href="/dps/fetch/">提取私密代理</a></li>
 63 |                 </ul>
 64 |             </li>
 65 |             <li id="menu_apidoc" class="dropdown"><a href="javascript:void(0);">API接口<b class="caret"></b></a>
 66 |                 <ul class="dropmenu">
 67 |                     <li><a href="/apidoc/">开放代理API</a></li>
 68 |                     <li><a href="/dps/apidoc/">私密代理API</a></li>
 69 |                 </ul>
 70 |             </li>
 71 |             <li id="menu_help"><a href="http://help.kuaidaili.com" target="_blank">帮助中心</a></li>
 72 |         </ul>
 73 |     </div>
 74 | 
 75 | </div>
 76 | 
 77 | 
 78 | 
 79 | 
 80 | <div class="taglineWrap">
 81 |     <div class="taglineContent">
 82 |         <h2>为什么使用快代理？</h2>
 83 |         <p>我们不间断地运行着极其高效、精准的公网代理收集系统，每天扫描的代理数以万计，最新出现的代理总能在第一时间收录。</p>
 84 |         <p>我们精确地检测每一个代理IP的匿名度、响应时间、数据传输速度、地域、运营商，平均每个IP每天被检测上百次，因此你总是可以在这里找到可以正常工作的代理。</p>
 85 |         <p>我们提供了极其丰富的代理筛选和API接口，只为更便捷地提取。</p>
 86 | 
 87 |     </div>
 88 | </div>
 89 | 
 90 | <div id="container">
 91 | 
 92 | 
 93 |     <div class="stat" style="margin: 20px 0;">
 94 |         <span>代理总数: <strong>15095323</strong>个<span class="hint"> 2分钟前更新</span></span>
 95 |         <span class="right">10分钟内可用: <strong>3911</strong>个<span class="hint"> 2分钟前更新</span></span>
 96 |     </div>
 97 | 
 98 |     <div id="freelist" style="margin:10px 0 10px 0;">
 99 |         <h3 style="text-align:center;">免费高速HTTP代理IP列表（2016-12-15）</h3>
100 |     </div>
101 |     <div id="intro">
102 |         <p></p>
103 |     </div>
104 |     <div id="index_free_list">
105 |         <table class="table table-bordered table-striped  table-index">
106 |             <thead>
107 |             <tr>
108 |                 <th>IP</th>
109 |                 <th>PORT</th>
110 |                 <th>匿名度</th>
111 |                 <th>类型</th>
112 |                 <th>get/post支持</th>
113 |                 <th>位置</th>
114 |                 <th>响应速度</th>
115 |                 <th>最后验证时间</th>
116 |             </tr>
117 |             </thead>
118 |             <tbody>
119 | 
120 |             <tr>
121 |                 <td data-title="IP">121.232.147.251</td>
122 |                 <td data-title="PORT">9000</td>
123 |                 <td data-title="匿名度">高匿名</td>
124 |                 <td data-title="类型">HTTP</td>
125 |                 <td data-title="get/post支持">GET, POST</td>
126 |                 <td data-title="位置">中国 江苏省 镇江市 电信</td>
127 |                 <td data-title="响应速度">3秒</td>
128 |                 <td data-title="最后验证时间">2分钟前</td>
129 |             </tr>
130 | 
131 |             <tr>
132 |                 <td data-title="IP">124.88.67.52</td>
133 |                 <td data-title="PORT">83</td>
134 |                 <td data-title="匿名度">高匿名</td>
135 |                 <td data-title="类型">HTTP</td>
136 |                 <td data-title="get/post支持">GET, POST</td>
137 |                 <td data-title="位置">中国 新疆维吾尔自治区 乌鲁木齐市 联通</td>
138 |                 <td data-title="响应速度">0.4秒</td>
139 |                 <td data-title="最后验证时间">2分钟前</td>
140 |             </tr>
141 | 
142 |             <tr>
143 |                 <td data-title="IP">183.196.170.247</td>
144 |                 <td data-title="PORT">9000</td>
145 |                 <td data-title="匿名度">透明</td>
146 |                 <td data-title="类型">HTTP, HTTPS</td>
147 |                 <td data-title="get/post支持">GET, POST</td>
148 |                 <td data-title="位置">中国 河北省 秦皇岛市 移动</td>
149 |                 <td data-title="响应速度">3秒</td>
150 |                 <td data-title="最后验证时间">5分钟前</td>
151 |             </tr>
152 | 
153 |             <tr>
154 |                 <td data-title="IP">221.193.235.90</td>
155 |                 <td data-title="PORT">9797</td>
156 |                 <td data-title="匿名度">透明</td>
157 |                 <td data-title="类型">HTTP, HTTPS</td>
158 |                 <td data-title="get/post支持">GET, POST</td>
159 |                 <td data-title="位置">中国 河北省 邯郸市 联通</td>
160 |                 <td data-title="响应速度">1秒</td>
161 |                 <td data-title="最后验证时间">8分钟前</td>
162 |             </tr>
163 | 
164 |             <tr>
165 |                 <td data-title="IP">121.232.145.88</td>
166 |                 <td data-title="PORT">9000</td>
167 |                 <td data-title="匿名度">高匿名</td>
168 |                 <td data-title="类型">HTTP</td>
169 |                 <td data-title="get/post支持">GET, POST</td>
170 |                 <td data-title="位置">中国 江苏省 镇江市 电信</td>
171 |                 <td data-title="响应速度">1秒</td>
172 |                 <td data-title="最后验证时间">11分钟前</td>
173 |             </tr>
174 | 
175 |             <tr>
176 |                 <td data-title="IP">117.90.2.31</td>
177 |                 <td data-title="PORT">9000</td>
178 |                 <td data-title="匿名度">高匿名</td>
179 |                 <td data-title="类型">HTTP</td>
180 |                 <td data-title="get/post支持">GET, POST</td>
181 |                 <td data-title="位置">中国 江苏省 镇江市 电信</td>
182 |                 <td data-title="响应速度">1秒</td>
183 |                 <td data-title="最后验证时间">14分钟前</td>
184 |             </tr>
185 | 
186 |             <tr>
187 |                 <td data-title="IP">117.90.1.121</td>
188 |                 <td data-title="PORT">9000</td>
189 |                 <td data-title="匿名度">高匿名</td>
190 |                 <td data-title="类型">HTTP</td>
191 |                 <td data-title="get/post支持">GET, POST</td>
192 |                 <td data-title="位置">中国 江苏省 镇江市 电信</td>
193 |                 <td data-title="响应速度">3秒</td>
194 |                 <td data-title="最后验证时间">17分钟前</td>
195 |             </tr>
196 | 
197 |             <tr>
198 |                 <td data-title="IP">121.232.147.52</td>
199 |                 <td data-title="PORT">9000</td>
200 |                 <td data-title="匿名度">高匿名</td>
201 |                 <td data-title="类型">HTTP</td>
202 |                 <td data-title="get/post支持">GET, POST</td>
203 |                 <td data-title="位置">中国 江苏省 镇江市 电信</td>
204 |                 <td data-title="响应速度">1秒</td>
205 |                 <td data-title="最后验证时间">20分钟前</td>
206 |             </tr>
207 | 
208 |             <tr>
209 |                 <td data-title="IP">121.61.248.221</td>
210 |                 <td data-title="PORT">9000</td>
211 |                 <td data-title="匿名度">高匿名</td>
212 |                 <td data-title="类型">HTTP</td>
213 |                 <td data-title="get/post支持">GET, POST</td>
214 |                 <td data-title="位置">中国 湖北省 荆门市 电信</td>
215 |                 <td data-title="响应速度">0.5秒</td>
216 |                 <td data-title="最后验证时间">23分钟前</td>
217 |             </tr>
218 | 
219 |             <tr>
220 |                 <td data-title="IP">106.2.187.202</td>
221 |                 <td data-title="PORT">8080</td>
222 |                 <td data-title="匿名度">透明</td>
223 |                 <td data-title="类型">HTTP, HTTPS</td>
224 |                 <td data-title="get/post支持">GET, POST</td>
225 |                 <td data-title="位置">中国 北京市 北京市 </td>
226 |                 <td data-title="响应速度">3秒</td>
227 |                 <td data-title="最后验证时间">26分钟前</td>
228 |             </tr>
229 | 
230 |             </tbody>
231 |         </table>
232 |         <p>注：表中响应速度是中国测速服务器的测试数据，仅供参考。响应速度根据你机器所在的地理位置不同而有差异。</p>
233 | 
234 |         <div id="listnav">
235 |             <ul>
236 |                 <li>第</li>
237 |                 <li><a id="p1" href="/proxylist/1/">1</a></li>
238 |                 <li><a id="p2" href="/proxylist/2/">2</a></li>
239 |                 <li><a id="p3" href="/proxylist/3/">3</a></li>
240 |                 <li><a id="p4" href="/proxylist/4/">4</a></li>
241 |                 <li><a id="p5" href="/proxylist/5/">5</a></li>
242 |                 <li><a id="p6" href="/proxylist/6/">6</a></li>
243 |                 <li><a id="p7" href="/proxylist/7/">7</a></li>
244 |                 <li><a id="p8" href="/proxylist/8/">8</a></li>
245 |                 <li><a id="p9" href="/proxylist/9/">9</a></li>
246 |                 <li><a id="p10" href="/proxylist/10/">10</a></li>
247 |                 <li>页</li>
248 |             </ul>
249 |         </div>
250 |     </div>
251 | 
252 |     <div class="btn center"><a id="tobuy" href="/free/">更多免费代理</a></div>
253 | 
254 | </div>
255 | 
256 | 
257 | 
258 | <div id="footer2">
259 |     <div class="container">
260 |         <div class="copyright">© 2013 - 2016 Kuaidaili.com 版权所有</div>
261 |         <div class="footnav"><a href="/about/">关于我们</a><span>|</span><a href="/privacy/">隐私政策</a><span>|</span><a href="http://blog.kuaidaili.com/" target="_blank">官方博客</a><span>|</span><a href="/sitemap.xml">网站地图</a><span>|</span><a href="http://help.kuaidaili.com" target="_blank">帮助中心</a></div>
262 |         <div class="icon">
263 |             <a href="https://ss.knet.cn/verifyseal.dll?sn=e161117110108652324qkr000000&ct=df&a=1&pa=0.3305956236561214" target="_blank" title="可信网站"><img height="40" src="http://img.kuaidaili.com/img/kxlogo.png" alt="可信网站" /></a>
264 |         </div>
265 |     </div>
266 | </div>
267 | 
268 | <ul class="onMShow" style="display: block;">
269 |     <li>
270 |         <a href="/support/"><br>
271 |             <span class="bt1"></span>
272 |             <div>提交工单</div>
273 |         </a>
274 |     </li>
275 |     <li>
276 |         <a href="tel:010-57216816"><br>
277 |             <span class="bt2"></span>
278 |             <div class="two">客服电话：010-57216816<br/>周一至周六 9:00-18:00</div>
279 |         </a>
280 |     </li>
281 |     <li>
282 |         <a href="http://wpa.qq.com/msgrd?v=3&uin=2162481273&site=qq&menu=yes"><br>
283 |             <span class="bt3"></span>
284 |             <div class="two">客服QQ：2162481273<br/>周一至周六 9:00-18:00</div>
285 |         </a>
286 |     </li>
287 | </ul>
288 | <a href="#top" id="top_btn" class="label btt" style="display:none;"><span class="icon-chevron-up icon-white"></span></a>
289 | 
290 | 
291 | <!--[if lt IE 8]><link rel="stylesheet" href="http://img.kuaidaili.com/css/ie.css?v=2" media="screen" /><![endif]-->
292 | <script language="javascript" type="text/javascript" src="http://img.kuaidaili.com/js/all.js?v=20"></script>
293 | 
294 | <script type="text/javascript">
295 | $("tbody tr:nth-child(odd)").addClass("odd");
296 | $(document).ready(function() {
297 |     if(window.location.pathname.indexOf("/proxylist/") != -1)
298 |         jumpToAnchor("freelist");
299 |     $("#p1").addClass("active");
300 | });
301 | </script>
302 | 
303 | <script type="text/javascript">
304 | 
305 | var menu = "menu_list";
306 | if(menu) $('#'+menu).addClass('active');
307 | var ucm = "";
308 | if(ucm){
309 |     $('#ucm_'+ucm).addClass('active');
310 |     $('#ucm_'+ucm+' a i').addClass('icon-white');
311 | }
312 | 
313 | var _hmt = _hmt || [];
314 | (function() {
315 |   var hm = document.createElement("script");
316 |   hm.src = "//hm.baidu.com/hm.js?7ed65b1cc4b810e9fd37959c9bb51b31";
317 |   var s = document.getElementsByTagName("script")[0];
318 |   s.parentNode.insertBefore(hm, s);
319 | })();
320 | 
321 | (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
322 | (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
323 | m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
324 | })(window,document,'script','http://img.kuaidaili.com/ga/analytics.js','ga');
325 | ga('create', 'UA-76097251-1', 'auto');
326 | ga('send', 'pageview');
327 | 
328 | </script>
329 | 
330 | <link rel="stylesheet" href="http://img.kuaidaili.com/css/base.min.css?v=105" media="screen" />
331 | </body>
332 | </html>
333 | 


--------------------------------------------------------------------------------