├── src ├── main │ ├── resources │ │ └── application.properties │ └── java │ │ └── com │ │ └── noisy │ │ └── phantom │ │ └── PhantomApplication.java └── test │ └── java │ └── com │ └── noisy │ └── phantom │ └── PhantomApplicationTests.java ├── proxy-grab ├── target │ ├── classes │ │ ├── META-INF │ │ │ └── proxy-grab.kotlin_module │ │ ├── com │ │ │ └── noisy │ │ │ │ ├── GrabCenter.class │ │ │ │ ├── entity │ │ │ │ └── AgentIp.class │ │ │ │ ├── utils │ │ │ │ ├── IPutils.class │ │ │ │ ├── SpringUtil.class │ │ │ │ ├── CodeGenerator.class │ │ │ │ ├── CodeGenerator$1.class │ │ │ │ ├── CodeGenerator$2.class │ │ │ │ └── SpringContextUtil.class │ │ │ │ ├── task │ │ │ │ ├── Upload5UTask.class │ │ │ │ ├── UploadXiciTask.class │ │ │ │ ├── UploadGoubanjiaTask.class │ │ │ │ └── UploadKuaidailiTask.class │ │ │ │ ├── mapper │ │ │ │ └── AgentIpMapper.class │ │ │ │ ├── service │ │ │ │ ├── IAgentIpService.class │ │ │ │ └── impl │ │ │ │ │ └── AgentIpServiceImpl.class │ │ │ │ └── controller │ │ │ │ └── AgentIpController.class │ │ ├── mapper │ │ │ └── AgentIpMapper.xml │ │ ├── application.yml │ │ └── spring.xml │ └── test-classes │ │ ├── META-INF │ │ └── proxy-grab.kotlin_module │ │ └── com │ │ └── noisy │ │ └── phantom │ │ ├── PhantomApplicationTests.class │ │ └── phantom │ │ └── PhantomApplicationTests.class ├── src │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── noisy │ │ │ │ ├── service │ │ │ │ ├── IAgentIpService.java │ │ │ │ └── impl │ │ │ │ │ └── AgentIpServiceImpl.java │ │ │ │ ├── controller │ │ │ │ └── AgentIpController.java │ │ │ │ ├── mapper │ │ │ │ └── AgentIpMapper.java │ │ │ │ ├── entity │ │ │ │ └── AgentIp.java │ │ │ │ ├── task │ │ │ │ ├── UploadGoubanjiaTask.java │ │ │ │ ├── Upload5UTask.java │ │ │ │ ├── UploadKuaidailiTask.java │ │ │ │ └── UploadXiciTask.java │ │ │ │ ├── GrabCenter.java │ │ │ │ └── utils │ │ │ │ ├── IPutils.java │ │ │ │ ├── SpringContextUtil.java │ │ │ │ ├── SpringUtil.java │ │ │ │ └── CodeGenerator.java │ │ └── resources │ │ │ ├── mapper │ │ │ └── AgentIpMapper.xml │ │ │ ├── agent_ip.sql │ │ │ └── spring.xml │ └── test │ │ └── java │ │ └── com │ │ └── noisy │ │ └── PhantomApplicationTest.java └── pom.xml ├── .gitattributes ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ ├── maven-wrapper.properties │ └── MavenWrapperDownloader.java ├── public_shell.md ├── phantom-backend ├── target │ └── test-classes │ │ └── com │ │ └── noisy │ │ └── AppTest.class ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── noisy │ │ │ └── App.java │ └── test │ │ └── java │ │ └── com │ │ └── noisy │ │ └── AppTest.java └── pom.xml ├── proxy-kidd ├── src │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── noisy │ │ │ │ └── proxy │ │ │ │ ├── HttpProxyDetect.java │ │ │ │ ├── entity │ │ │ │ ├── ProtocolType.java │ │ │ │ ├── ProxyType.java │ │ │ │ ├── InputType.java │ │ │ │ ├── ProxyIp.java │ │ │ │ └── ProxyInfo.java │ │ │ │ ├── repository │ │ │ │ └── ProxyIpRepository.java │ │ │ │ ├── dao │ │ │ │ ├── ProxyInfoDao.java │ │ │ │ ├── AbstractProxyInfoDaoImpl.java │ │ │ │ └── ProxyInfoDaoFilempl.java │ │ │ │ ├── detector │ │ │ │ ├── ProxyDetector.java │ │ │ │ ├── ProxyDetectorFactory.java │ │ │ │ ├── AbstractProxyDetector.java │ │ │ │ ├── HTTPProxyDetector.java │ │ │ │ ├── Socks4ProxyDetector.java │ │ │ │ └── Socks5ProxyDetector.java │ │ │ │ ├── ProxyProbApplication.java │ │ │ │ ├── service │ │ │ │ └── ProxyIpService.java │ │ │ │ ├── util │ │ │ │ ├── IPLocationUtils.java │ │ │ │ ├── SpringContextUtils.java │ │ │ │ ├── IPFilterUtils.java │ │ │ │ ├── IPSegment.java │ │ │ │ └── IPPoolUtils.java │ │ │ │ ├── EmailReporter.java │ │ │ │ ├── TaskScheduler.java │ │ │ │ └── ProxyScanner.java │ │ └── resources-env │ │ │ ├── prod │ │ │ ├── application.yml │ │ │ └── logback.xml │ │ │ └── dev │ │ │ ├── application.yml │ │ │ └── logback.xml │ └── test │ │ └── java │ │ └── com │ │ └── noisy │ │ └── proxy │ │ └── AppTest.java ├── restart.sh └── pom.xml ├── phantom-master ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── noisy │ │ │ └── App.java │ └── test │ │ └── java │ │ └── com │ │ └── noisy │ │ └── AppTest.java └── pom.xml ├── README.md ├── .github └── FUNDING.yml ├── .gitignore ├── todo.md ├── pom.xml ├── mvnw.cmd └── mvnw /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /proxy-grab/target/classes/META-INF/proxy-grab.kotlin_module: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /proxy-grab/target/test-classes/META-INF/proxy-grab.kotlin_module: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /public_shell.md: -------------------------------------------------------------------------------- 1 | - 子模块生成命令 2 | `mvn archetype:generate -DgroupId=com.noisy -DartifactId=proxy-kidd -DinteractiveMode=false` 3 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip 2 | -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/GrabCenter.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/GrabCenter.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/entity/AgentIp.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/entity/AgentIp.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/utils/IPutils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/utils/IPutils.class -------------------------------------------------------------------------------- /phantom-backend/target/test-classes/com/noisy/AppTest.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/phantom-backend/target/test-classes/com/noisy/AppTest.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/task/Upload5UTask.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/task/Upload5UTask.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/utils/SpringUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/utils/SpringUtil.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/mapper/AgentIpMapper.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/mapper/AgentIpMapper.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/task/UploadXiciTask.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/task/UploadXiciTask.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/utils/CodeGenerator.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/utils/CodeGenerator.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/utils/CodeGenerator$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/utils/CodeGenerator$1.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/utils/CodeGenerator$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/utils/CodeGenerator$2.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/service/IAgentIpService.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/service/IAgentIpService.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/task/UploadGoubanjiaTask.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/task/UploadGoubanjiaTask.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/task/UploadKuaidailiTask.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/task/UploadKuaidailiTask.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/utils/SpringContextUtil.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/utils/SpringContextUtil.class -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/controller/AgentIpController.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/controller/AgentIpController.class -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/HttpProxyDetect.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy; 2 | 3 | /** 4 | * @author lei.X 5 | * @date 2019/4/29 6 | */ 7 | public class HttpProxyDetect { 8 | } 9 | -------------------------------------------------------------------------------- /proxy-grab/target/classes/com/noisy/service/impl/AgentIpServiceImpl.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/classes/com/noisy/service/impl/AgentIpServiceImpl.class -------------------------------------------------------------------------------- /proxy-grab/target/test-classes/com/noisy/phantom/PhantomApplicationTests.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/test-classes/com/noisy/phantom/PhantomApplicationTests.class -------------------------------------------------------------------------------- /proxy-grab/target/test-classes/com/noisy/phantom/phantom/PhantomApplicationTests.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Metatronxl/phantom/HEAD/proxy-grab/target/test-classes/com/noisy/phantom/phantom/PhantomApplicationTests.class -------------------------------------------------------------------------------- /phantom-master/src/main/java/com/noisy/App.java: -------------------------------------------------------------------------------- 1 | package com.noisy; 2 | 3 | /** 4 | * Hello world! 5 | * 6 | */ 7 | public class App 8 | { 9 | public static void main( String[] args ) 10 | { 11 | System.out.println( "Hello World!" ); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /phantom-backend/src/main/java/com/noisy/App.java: -------------------------------------------------------------------------------- 1 | package com.noisy; 2 | 3 | 4 | /** 5 | * Hello world! 6 | * 7 | */ 8 | public class App 9 | { 10 | public static void main( String[] args ) 11 | { 12 | System.out.println( "Hello World!" ); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # phantom 2 | 3 | 全网环境下的代理扫描工具 4 | 5 | ## 功能 6 | - 支持对IPv4的全部扫描、v6地址的指定网段扫描 7 | - 支持指定http、https、sock4、sock5协议的扫描 8 | - 支持自定义ip列表和port的定向扫描 9 | - 提供常见ip代理服务网站的爬取功能，并录入高频ip库 10 | - 对录入数据库的代理ip进行定期验证 11 | 12 | ## TODO 13 | 14 | - 对录入数据库的ip设置打分机制 15 | - 针对代理需求给出代理接口 16 | 17 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/service/IAgentIpService.java: -------------------------------------------------------------------------------- 1 | package com.noisy.service; 2 | 3 | import com.noisy.entity.AgentIp; 4 | import com.baomidou.mybatisplus.extension.service.IService; 5 | 6 | /** 7 | *

8 | * 服务类 9 | *

10 | * 11 | * @author liuzhaoce 12 | * @since 2019-05-05 13 | */ 14 | public interface IAgentIpService extends IService { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/entity/ProtocolType.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.entity; 2 | 3 | /** 4 | * Created by kevin on 5/24/16. 5 | */ 6 | public enum ProtocolType { 7 | HTTP(0), 8 | SOCKS_V4(1), 9 | SOCKS_V5(2); 10 | 11 | private int type; 12 | 13 | ProtocolType(int type) { 14 | this.type = type; 15 | } 16 | 17 | public int getType() { 18 | return type; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/repository/ProxyIpRepository.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.repository; 2 | 3 | import com.noisy.proxy.entity.ProxyIp; 4 | import org.springframework.data.jpa.repository.JpaRepository; 5 | 6 | /** 7 | * @author lei.X 8 | * @date 2019/5/27 9 | */ 10 | public interface ProxyIpRepository extends JpaRepository{ 11 | 12 | 13 | ProxyIp findByIpAndPort(String ip,int port); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/dao/ProxyInfoDao.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.dao; 2 | 3 | import com.noisy.proxy.entity.ProxyInfo; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Created by kevin on 5/31/16. 9 | */ 10 | public interface ProxyInfoDao { 11 | 12 | public void insert(ProxyInfo proxyInfo); 13 | 14 | public void deleteByIPAndPort(String ip, int port); 15 | 16 | public void append(ProxyInfo proxyInfo, File distFile); 17 | } 18 | -------------------------------------------------------------------------------- /src/test/java/com/noisy/phantom/PhantomApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.noisy.phantom; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class PhantomApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | custom: # Replace with a single custom sponsorship URL 9 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/controller/AgentIpController.java: -------------------------------------------------------------------------------- 1 | package com.noisy.controller; 2 | 3 | 4 | import org.springframework.web.bind.annotation.RequestMapping; 5 | 6 | import org.springframework.web.bind.annotation.RestController; 7 | 8 | /** 9 | *

10 | * 前端控制器 11 | *

12 | * 13 | * @author liuzhaoce 14 | * @since 2019-05-05 15 | */ 16 | @RestController 17 | @RequestMapping("/agent-ip") 18 | public class AgentIpController { 19 | 20 | } 21 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/entity/ProxyType.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.entity; 2 | 3 | /** 4 | * @author lei.X 5 | * @date 2019/4/29 6 | */ 7 | public enum ProxyType { 8 | TRANSPARENT(0), 9 | ANONYMOUS(1), 10 | DISTORTING(2), 11 | HIGH_ANONYMOUS(3); 12 | 13 | private int type; 14 | 15 | ProxyType(int type) { 16 | this.type = type; 17 | } 18 | 19 | public int getType() { 20 | return type; 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/detector/ProxyDetector.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.detector; 2 | 3 | /** 4 | * Created by lei.x on 5/24/19.. 5 | */ 6 | public interface ProxyDetector { 7 | /** 8 | * Detect the given IP and port, will return the proxy info if its a proxy. 9 | * 10 | * @param ip 11 | * @param port 12 | */ 13 | void detect(String ip, int port); 14 | 15 | /** 16 | * Reset the proxy detector. 17 | */ 18 | void reset(); 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/noisy/phantom/PhantomApplication.java: -------------------------------------------------------------------------------- 1 | package com.noisy.phantom; 2 | 3 | import org.mybatis.spring.annotation.MapperScan; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | 7 | 8 | @SpringBootApplication 9 | @MapperScan("com.noisy") 10 | public class PhantomApplication { 11 | 12 | public static void main(String[] args) { 13 | SpringApplication.run(PhantomApplication.class, args); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/mapper/AgentIpMapper.java: -------------------------------------------------------------------------------- 1 | package com.noisy.mapper; 2 | 3 | import com.noisy.entity.AgentIp; 4 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 5 | import org.apache.ibatis.annotations.Mapper; 6 | 7 | /** 8 | *

9 | * Mapper 接口 10 | *

11 | * 12 | * @author liuzhaoce 13 | * @since 2019-05-05 14 | */ 15 | 16 | @Mapper 17 | public interface AgentIpMapper extends BaseMapper { 18 | 19 | int insertAgentIp(AgentIp agentIp); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/entity/InputType.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.entity; 2 | 3 | /** 4 | * Created by kevin on 8/22/16. 5 | */ 6 | public enum InputType { 7 | IP_LIST("ip_list"), 8 | IP_SEGMENTS("ip_segments"); 9 | 10 | InputType(String type){ 11 | this.type = type; 12 | } 13 | 14 | private String type; 15 | 16 | public String getType() { 17 | return type; 18 | } 19 | 20 | public boolean isThisType(String type){ 21 | return this.type.equals(type); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/dao/AbstractProxyInfoDaoImpl.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.dao; 2 | 3 | import com.noisy.proxy.entity.ProxyInfo; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Created by kevin on 5/31/16. 9 | */ 10 | public abstract class AbstractProxyInfoDaoImpl implements ProxyInfoDao { 11 | 12 | @Override 13 | public void insert(ProxyInfo proxyInfo) { 14 | } 15 | 16 | @Override 17 | public void deleteByIPAndPort(String ip, int port) { 18 | } 19 | 20 | @Override 21 | public void append(ProxyInfo proxyInfo, File distFile) { 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | /target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | 5 | ### STS ### 6 | .apt_generated 7 | .classpath 8 | .factorypath 9 | .project 10 | .settings 11 | .springBeans 12 | .sts4-cache 13 | 14 | ### IntelliJ IDEA ### 15 | .idea 16 | *.iws 17 | *.iml 18 | *.ipr 19 | 20 | ### NetBeans ### 21 | /nbproject/private/ 22 | /nbbuild/ 23 | /dist/ 24 | /nbdist/ 25 | /.nb-gradle/ 26 | /build/ 27 | 28 | 29 | ### VS Code ### 30 | .vscode/ 31 | 32 | ### Others ### 33 | commandIgnore.md 34 | /services 35 | /output 36 | proxy-kidd/target/* 37 | proxy-grab/target/* 38 | phantom-backend/target/classes/ 39 | phantom-backend/target/generated-sources/ 40 | phantom-backend/target/generated-test-sources/ 41 | proxy-grab/src/main/resources/application.yml -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | ### TODO 2 | 3 | - 分发扫描任务：主节点负责管理和分发ip扫描任务，salve节点负责连接主节点和领取任务 4 | ```txt 5 | 初步构想是salve通过netty连接到master， 6 | master检测到有slave连接后会自动分发任务， 7 | slave完成后返回响应 8 | 9 | 一个IP segment 作为一个task由master进行分发 10 | 11 | master 使用定时任务扫描是否有连接上的slave机器 12 | 13 | salve 扫描完成后返回给master 结果， 数据则直接存贮进数据库 14 | 15 | 16 | 任务分发的数据库采用redis，所有IP_segment 放入list消息队列 17 | 18 | 19 | 20 | 21 | 现在的思路是phantom 分为phantom-master和phantom-salve， phantom-master专门做任务的分发和 22 | IP打分这类的工作，phantom-master将IP-segment存储进redis的消息队列（一个段/24作为一条数据），有salve连接上了就会分发 23 | 一个segment给它。 24 | 25 | salve和master通过netty进行通讯，这样可以动态的增减salve的个数 26 | 27 | 28 | 29 | ``` 30 | 31 | - IP打分机制 32 | - 开放使用端口 33 | 34 | - 数据库添加扫描的代理类型 35 | 36 | - 压测机器最多可以开多少个Channel -------------------------------------------------------------------------------- /phantom-backend/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.noisy 7 | phantom 8 | 0.0.1-SNAPSHOT 9 | 10 | com.noisy 11 | phantom-backend 12 | phantom-backend 13 | http://maven.apache.org 14 | 15 | 16 | 17 | com.noisy 18 | proxy-kidd 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/ProxyProbApplication.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.ConfigurableApplicationContext; 6 | import org.springframework.data.jpa.repository.config.EnableJpaAuditing; 7 | 8 | /** 9 | * @author lei.X 10 | * @date 2019/4/29 11 | */ 12 | @SpringBootApplication 13 | @EnableJpaAuditing 14 | public class ProxyProbApplication { 15 | 16 | public static void main(String[] args) { 17 | ConfigurableApplicationContext context = SpringApplication.run(ProxyProbApplication.class, args); 18 | ProxyScanner bean = context.getBean(ProxyScanner.class); 19 | 20 | bean.start(); 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/entity/AgentIp.java: -------------------------------------------------------------------------------- 1 | package com.noisy.entity; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.EqualsAndHashCode; 6 | import lombok.experimental.Accessors; 7 | 8 | import java.io.Serializable; 9 | 10 | /** 11 | *

12 | * 13 | *

14 | * 15 | * @author liuzhaoce 16 | * @since 2019-05-05 17 | */ 18 | @Data 19 | @EqualsAndHashCode(callSuper = false) 20 | @Accessors(chain = true) 21 | @AllArgsConstructor 22 | public class AgentIp implements Serializable { 23 | 24 | private static final long serialVersionUID = 1L; 25 | 26 | private String ip; 27 | 28 | private String port; 29 | 30 | private String anonymity; 31 | 32 | private String type; 33 | 34 | private String location; 35 | 36 | private String operator; 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /phantom-backend/src/test/java/com/noisy/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.noisy; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /phantom-master/src/test/java/com/noisy/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.noisy; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /proxy-grab/target/classes/mapper/AgentIpMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | insert into agent_ip( 7 | ip, 8 | port, 9 | anonymity, 10 | type, 11 | location, 12 | operator 13 | ) values( 14 | #{ip}, 15 | #{port}, 16 | #{anonymity}, 17 | #{type}, 18 | #{location}, 19 | #{operator} 20 | ) on duplicate key update 21 | anonymity = #{anonymity}, 22 | type = #{type}, 23 | location = #{location}, 24 | operator = #{operator} 25 | 26 | 27 | -------------------------------------------------------------------------------- /proxy-kidd/src/test/java/com/noisy/proxy/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /proxy-grab/src/main/resources/mapper/AgentIpMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | insert into agent_ip( 7 | ip, 8 | port, 9 | anonymity, 10 | type, 11 | location, 12 | operator 13 | ) values( 14 | #{ip}, 15 | #{port}, 16 | #{anonymity}, 17 | #{type}, 18 | #{location}, 19 | #{operator} 20 | ) on duplicate key update 21 | anonymity = #{anonymity}, 22 | type = #{type}, 23 | location = #{location}, 24 | operator = #{operator} 25 | 26 | 27 | -------------------------------------------------------------------------------- /phantom-master/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.noisy 7 | phantom 8 | 0.0.1-SNAPSHOT 9 | 10 | com.noisy 11 | phantom-master 12 | 1.0-SNAPSHOT 13 | phantom-master 14 | http://maven.apache.org 15 | 16 | 17 | junit 18 | junit 19 | 3.8.1 20 | test 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /proxy-grab/target/classes/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 2525 3 | 4 | # 该配置的名称是固定的，不可以改变，否则将不能自动加载到数据源中 5 | spring: 6 | datasource: 7 | # 使用druid数据源 8 | type: com.alibaba.druid.pool.DruidDataSource 9 | driver-class-name: com.mysql.cj.jdbc.Driver 10 | url: jdbc:mysql://139.224.187.178:3306/phantom?characterEncoding=utf-8&autoReconnect=true&useSSL=false 11 | username: root 12 | password: phantom 13 | filters: stat 14 | maxActive: 20 15 | initialSize: 1 16 | maxWait: 60000 17 | minIdle: 1 18 | timeBetweenEvictionRunsMillis: 60000 19 | minEvictableIdleTimeMillis: 300000 20 | validationQuery: select 1 FROM DUAL 21 | testWhileIdle: true 22 | testOnBorrow: false 23 | testOnReturn: false 24 | poolPreparedStatements: true 25 | maxOpenPreparedStatements: 20 26 | 27 | mybatis-plus: 28 | mapper-locations: classpath:/mapper/*Mapper.xml -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/task/UploadGoubanjiaTask.java: -------------------------------------------------------------------------------- 1 | package com.noisy.task; 2 | 3 | import com.noisy.service.impl.AgentIpServiceImpl; 4 | import com.virjar.dungproxy.client.httpclient.HttpInvoker; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.scheduling.annotation.EnableScheduling; 7 | import org.springframework.stereotype.Component; 8 | 9 | import javax.annotation.Resource; 10 | 11 | /** 12 | * @Auther: liuzhaoce 13 | * @Date: 2019-05-05 17:13 14 | * @Description: 15 | */ 16 | 17 | @Component 18 | @Configuration 19 | @EnableScheduling 20 | public class UploadGoubanjiaTask { 21 | 22 | @Resource 23 | AgentIpServiceImpl agentIpService; 24 | 25 | // @Scheduled(fixedRate = 5000) 26 | private void configureTasks() { 27 | 28 | String s = HttpInvoker.get("http://www.goubanjia.com/"); 29 | 30 | // TODO : 爬取goubanjia的代理ip，页面分析较为繁琐，每个数字放在不同的标签里 31 | 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/service/impl/AgentIpServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.noisy.service.impl; 2 | 3 | import com.noisy.entity.AgentIp; 4 | import com.noisy.mapper.AgentIpMapper; 5 | import com.noisy.service.IAgentIpService; 6 | import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; 7 | import org.springframework.stereotype.Service; 8 | 9 | import javax.annotation.Resource; 10 | import java.util.List; 11 | 12 | /** 13 | *

14 | * 服务实现类 15 | *

16 | * 17 | * @author liuzhaoce 18 | * @since 2019-05-05 19 | */ 20 | @Service 21 | public class AgentIpServiceImpl extends ServiceImpl implements IAgentIpService { 22 | 23 | @Resource 24 | private AgentIpMapper agentIpMapper; 25 | 26 | 27 | public void upload(List agentIpList) { 28 | for (AgentIp agentIp : agentIpList) { 29 | int code = agentIpMapper.insertAgentIp(agentIp); 30 | // log.info("return code: " + code); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /proxy-grab/src/test/java/com/noisy/PhantomApplicationTest.java: -------------------------------------------------------------------------------- 1 | //package com.noisy; 2 | // 3 | //import com.noisy.entity.AgentIp; 4 | //import com.noisy.mapper.AgentIpMapper; 5 | //import org.junit.Test; 6 | //import org.junit.runner.RunWith; 7 | //import org.springframework.beans.factory.annotation.Autowired; 8 | //import org.springframework.boot.test.context.SpringBootTest; 9 | //import org.springframework.test.context.junit4.SpringRunner; 10 | // 11 | //import java.util.List; 12 | // 13 | // 14 | //@RunWith(SpringRunner.class) 15 | //@SpringBootTest(classes = GrabCenter.class) 16 | //public class PhantomApplicationTest { 17 | // 18 | // @Autowired 19 | // private AgentIpMapper agentIpMapper; 20 | // 21 | // @Test 22 | // public void testSelect() { 23 | // System.out.println(("----- selectAll method test ------")); 24 | // List agentIpList = agentIpMapper.selectList(null); 25 | //// Assert.assertEquals(5, agentIpList.size()); 26 | // agentIpList.forEach(System.out::println); 27 | // } 28 | //} -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/detector/ProxyDetectorFactory.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.detector; 2 | 3 | import com.noisy.proxy.TaskScheduler; 4 | import com.noisy.proxy.entity.ProtocolType; 5 | 6 | /** 7 | * Created by lei.x on 5/24/19. 8 | */ 9 | public class ProxyDetectorFactory { 10 | /** 11 | * Create a new proxy detector with the given protocol type and task scheduler. 12 | * 13 | * @param protocolType ProtocolType 14 | * @param scheduler TaskSchedule Object 15 | * @return a ProxyDetector Object 16 | */ 17 | public static ProxyDetector createProxyDetector(ProtocolType protocolType, TaskScheduler scheduler) { 18 | switch (protocolType) { 19 | case HTTP: 20 | return new HTTPProxyDetector(scheduler); 21 | case SOCKS_V4: 22 | return new Socks4ProxyDetector(scheduler); 23 | case SOCKS_V5: 24 | return new Socks5ProxyDetector(scheduler); 25 | } 26 | 27 | return new HTTPProxyDetector(scheduler); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/GrabCenter.java: -------------------------------------------------------------------------------- 1 | package com.noisy; 2 | 3 | 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.mybatis.spring.annotation.MapperScan; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.boot.builder.SpringApplicationBuilder; 9 | import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; 10 | import org.springframework.context.ConfigurableApplicationContext; 11 | 12 | /** 13 | * @Auther: liuzhaoce 14 | * @Date: 2019-04-26 15:46 15 | * @Description: 16 | */ 17 | 18 | @SpringBootApplication 19 | @MapperScan("com.noisy") 20 | @Slf4j 21 | public class GrabCenter extends SpringBootServletInitializer { 22 | 23 | public static void main(String[] args) { 24 | 25 | //未使用，留下自建类使用Spring模块的方式 26 | ConfigurableApplicationContext context = SpringApplication.run(GrabCenter.class, args); 27 | } 28 | 29 | @Override 30 | protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) { 31 | return builder.sources(GrabCenter.class); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/utils/IPutils.java: -------------------------------------------------------------------------------- 1 | package com.noisy.utils; 2 | 3 | import com.google.common.collect.Lists; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @Auther: liuzhaoce 9 | * @Date: 2019-04-28 17:22 10 | * @Description: 11 | */ 12 | public class IPutils { 13 | 14 | 15 | /** 16 | * @Description: 拆分List 17 | * @param size 拆分成的每个List的大小 18 | * @param target 待拆分List 19 | */ 20 | public static List> createList(List target, int size) { 21 | 22 | List> listArr = Lists.newArrayList(); 23 | //获取被拆分的数组个数 24 | int arrSize = target.size() % size == 0 ? target.size() / size : target.size() / size + 1; 25 | for (int i = 0; i < arrSize; i++) { 26 | List sub = Lists.newArrayList(); 27 | //把指定索引数据放入到list中 28 | for (int j = i * size; j <= size * (i + 1) - 1; j++) { 29 | if (j <= target.size() - 1) { 30 | sub.add(target.get(j)); 31 | } 32 | } 33 | listArr.add(sub); 34 | } 35 | return listArr; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /proxy-grab/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.noisy 7 | phantom 8 | 0.0.1-SNAPSHOT 9 | 10 | com.noisy 11 | proxy-grab 12 | proxy-grab 13 | http://maven.apache.org 14 | 15 | 16 | junit 17 | junit 18 | 3.8.1 19 | test 20 | 21 | 22 | com.virjar 23 | dungproxy-client 24 | 0.0.8 25 | 26 | 27 | junit 28 | junit 29 | test 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/entity/ProxyIp.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.entity; 2 | 3 | import lombok.Data; 4 | import org.springframework.data.annotation.CreatedDate; 5 | import org.springframework.data.annotation.LastModifiedDate; 6 | import org.springframework.data.jpa.domain.support.AuditingEntityListener; 7 | 8 | import javax.persistence.*; 9 | import java.util.Date; 10 | 11 | /** 12 | * @author lei.X 13 | * @date 2019/5/27 14 | */ 15 | @Entity 16 | @Data 17 | @Table(name = "proxy_ip") 18 | @EntityListeners(AuditingEntityListener.class) 19 | public class ProxyIp { 20 | 21 | @Id 22 | @GeneratedValue 23 | private Long id; 24 | @Column(nullable = false) 25 | private String ip; 26 | @Column 27 | private int protocol; 28 | @Column(nullable = false) 29 | private int port; 30 | @Column(nullable = false) 31 | private int type; 32 | @Column 33 | private String location; 34 | 35 | /** 36 | * 创建时间 37 | */ 38 | @CreatedDate 39 | @Column(name = "create_time") 40 | private Date createTime; 41 | 42 | /** 43 | * 修改时间 44 | */ 45 | @LastModifiedDate 46 | @Column(name = "modify_time") 47 | private Date modifyTime; 48 | } 49 | -------------------------------------------------------------------------------- /proxy-grab/src/main/resources/agent_ip.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat Premium Data Transfer 3 | 4 | Source Server : proxy_grab 5 | Source Server Type : MySQL 6 | Source Server Version : 80016 7 | Source Host : localhost:3306 8 | Source Schema : proxy_grab 9 | 10 | Target Server Type : MySQL 11 | Target Server Version : 80016 12 | File Encoding : 65001 13 | 14 | Date: 06/05/2019 15:37:53 15 | */ 16 | 17 | SET NAMES utf8mb4; 18 | SET FOREIGN_KEY_CHECKS = 0; 19 | 20 | -- ---------------------------- 21 | -- Table structure for agent_ip 22 | -- ---------------------------- 23 | DROP TABLE IF EXISTS `agent_ip`; 24 | CREATE TABLE `agent_ip` ( 25 | `ip` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 26 | `port` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, 27 | `anonymity` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, 28 | `type` varchar(255) DEFAULT NULL, 29 | `location` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, 30 | `operator` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, 31 | PRIMARY KEY (`port`,`ip`) USING BTREE 32 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 33 | 34 | SET FOREIGN_KEY_CHECKS = 1; 35 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/dao/ProxyInfoDaoFilempl.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.dao; 2 | 3 | import com.noisy.proxy.entity.ProxyInfo; 4 | import com.noisy.proxy.entity.ProxyIp; 5 | import com.noisy.proxy.repository.ProxyIpRepository; 6 | import com.noisy.proxy.service.ProxyIpService; 7 | import com.noisy.proxy.util.SpringContextUtils; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.apache.commons.io.FileUtils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.stereotype.Component; 13 | 14 | import java.io.File; 15 | import java.io.IOException; 16 | 17 | /** 18 | * Created by kevin on 5/31/16. 19 | */ 20 | 21 | @Slf4j 22 | public class ProxyInfoDaoFilempl extends AbstractProxyInfoDaoImpl { 23 | 24 | 25 | ProxyIpService proxyIpService = (ProxyIpService) SpringContextUtils.getApplicationContext().getBean("proxyIpService"); 26 | 27 | @Override 28 | public void append(ProxyInfo proxyInfo, File distFile) { 29 | try { 30 | FileUtils.write(distFile, proxyInfo.toString(), "UTF-8", true); 31 | proxyIpService.saveNewProxyIp(proxyInfo); 32 | } catch (IOException e) { 33 | log.error("An exception occurred when appending a new proxy info to the file: {}", 34 | distFile.getAbsolutePath()); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/service/ProxyIpService.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.service; 2 | 3 | import com.noisy.proxy.entity.ProxyInfo; 4 | import com.noisy.proxy.entity.ProxyIp; 5 | import com.noisy.proxy.repository.ProxyIpRepository; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.stereotype.Service; 8 | 9 | import javax.annotation.Resource; 10 | import javax.transaction.Transactional; 11 | 12 | /** 13 | * @author lei.X 14 | * @date 2019/5/27 15 | */ 16 | @Service("proxyIpService") 17 | @Slf4j 18 | @Transactional 19 | public class ProxyIpService { 20 | 21 | @Resource 22 | private ProxyIpRepository proxyIpRepository; 23 | 24 | 25 | /** 26 | * 如果已经存在IP与port相同的数据，则不重复插入 27 | * @param proxyInfo 28 | */ 29 | public void saveNewProxyIp(ProxyInfo proxyInfo){ 30 | 31 | 32 | String ip = proxyInfo.getIp(); 33 | int port = proxyInfo.getPort(); 34 | 35 | 36 | ProxyIp dbProxy = proxyIpRepository.findByIpAndPort(ip,port); 37 | if (dbProxy == null){ 38 | ProxyIp proxyIp = new ProxyIp(); 39 | proxyIp.setIp(proxyInfo.getIp()); 40 | proxyIp.setPort(proxyInfo.getPort()); 41 | proxyIp.setType(proxyInfo.getType()); 42 | proxyIp.setLocation(proxyInfo.getLocation()); 43 | proxyIp.setProtocol(proxyInfo.getProtocol()); 44 | proxyIpRepository.save(proxyIp); 45 | log.info("new proxyInfo insert into database success"); 46 | }else{ 47 | log.warn("ip-port has been insert into database, so refuse insert"); 48 | } 49 | } 50 | 51 | 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/util/IPLocationUtils.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.util; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import us.monoid.web.JSONResource; 6 | import us.monoid.web.Resty; 7 | 8 | /** 9 | * Created by kevin on 5/27/16. 10 | */ 11 | public class IPLocationUtils { 12 | private final static Logger log = LoggerFactory.getLogger(IPLocationUtils.class); 13 | private final static Resty restClient = new Resty(); 14 | private final static String geoURL ="http://maxcd03-dev05/geoip/ip2coor/"; 15 | 16 | public static String getLocation(final String ip) { 17 | String location = null; 18 | String url = geoURL + ip; 19 | try { 20 | JSONResource response = restClient.json(url); 21 | location = constructLocation(response); 22 | } catch (Exception e) { 23 | log.warn("Failed to retrieve the IP location from GEO service: {}", url); 24 | } 25 | 26 | return location; 27 | } 28 | 29 | private static String constructLocation(JSONResource resource) throws Exception { 30 | if (resource == null) { 31 | return null; 32 | } 33 | String country = resource.get("country").toString(); 34 | if (country == null) { 35 | country = ""; 36 | } 37 | String province = resource.get("province").toString(); 38 | if (province == null) { 39 | province = ""; 40 | } 41 | String city = resource.get("city").toString(); 42 | 43 | StringBuilder location = new StringBuilder(country); 44 | if (!country.equals(province)) { 45 | location.append(province); 46 | } 47 | if (!province.equals(city)) { 48 | location.append(city); 49 | } 50 | 51 | return location.toString(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/util/SpringContextUtils.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.util; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationContextAware; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * @author lei.X 10 | * @date 2019/5/24 11 | */ 12 | @Component 13 | public class SpringContextUtils implements ApplicationContextAware { 14 | 15 | /** 16 | * 上下文对象实例 17 | */ 18 | private static ApplicationContext applicationContext; 19 | 20 | @Override 21 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 22 | if (SpringContextUtils.applicationContext == null) { 23 | SpringContextUtils.applicationContext = applicationContext; 24 | } 25 | } 26 | 27 | /** 28 | * 获取applicationContext 29 | * 30 | * @return 31 | */ 32 | public static ApplicationContext getApplicationContext() { 33 | return applicationContext; 34 | } 35 | 36 | /** 37 | * 通过name获取 Bean. 38 | * 39 | * @param name 40 | * @return 41 | */ 42 | public static Object getBean(String name) { 43 | return getApplicationContext().getBean(name); 44 | } 45 | 46 | /** 47 | * 通过class获取Bean. 48 | * 49 | * @param clazz 50 | * @param 51 | * @return 52 | */ 53 | public static T getBean(Class clazz) { 54 | return getApplicationContext().getBean(clazz); 55 | } 56 | 57 | /** 58 | * 通过name,以及Clazz返回指定的Bean 59 | * 60 | * @param name 61 | * @param clazz 62 | * @param 63 | * @return 64 | */ 65 | public static T getBean(String name, Class clazz) { 66 | return getApplicationContext().getBean(name, clazz); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/utils/SpringContextUtil.java: -------------------------------------------------------------------------------- 1 | package com.noisy.utils; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 5 | import org.springframework.context.ApplicationContext; 6 | import org.springframework.context.ApplicationContextAware; 7 | 8 | /** 9 | * @Auther: liuzhaoce 10 | * @Date: 2019-04-29 17:03 11 | * @Description: 12 | */ 13 | 14 | public class SpringContextUtil implements ApplicationContextAware { 15 | 16 | private static ApplicationContext applicationContext; // Spring应用上下文环境 17 | 18 | // 下面的这个方法上加了@Override注解，原因是继承ApplicationContextAware接口是必须实现的方法 19 | @Override 20 | public void setApplicationContext(ApplicationContext applicationContext) 21 | throws BeansException { 22 | SpringContextUtil.applicationContext = applicationContext; 23 | } 24 | 25 | public static ApplicationContext getApplicationContext() { 26 | return applicationContext; 27 | } 28 | 29 | public static Object getBean(String name) throws BeansException { 30 | return applicationContext.getBean(name); 31 | } 32 | 33 | public static Object getBean(String name, Class requiredType) 34 | throws BeansException { 35 | return applicationContext.getBean(name, requiredType); 36 | } 37 | 38 | public static boolean containsBean(String name) { 39 | return applicationContext.containsBean(name); 40 | } 41 | 42 | public static boolean isSingleton(String name) throws NoSuchBeanDefinitionException { 43 | return applicationContext.isSingleton(name); 44 | } 45 | 46 | public static Class getType(String name) throws NoSuchBeanDefinitionException { 47 | return applicationContext.getType(name); 48 | } 49 | 50 | public static String[] getAliases(String name) throws NoSuchBeanDefinitionException { 51 | return applicationContext.getAliases(name); 52 | } 53 | } -------------------------------------------------------------------------------- /proxy-kidd/restart.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | PROJECT_NAME="proxy-kidd" 5 | JAR_NAME="proxy-kidd-1.0-SNAPSHOT.jar" 6 | 7 | 8 | function compile() 9 | { 10 | echo "rebuild project" 11 | mvn clean package -Pprod -Dmaven.test.skip=true 12 | } 13 | 14 | function hasCodeUpdate() 15 | { 16 | git fetch origin master 17 | diff=`git diff FETCH_HEAD` 18 | if [ ! -n "$diff" ] ;then 19 | return 0 20 | fi 21 | echo "服务器代码更新" 22 | return 1 23 | } 24 | 25 | function pullNewCode() 26 | { 27 | git fetch origin master 28 | git merge 29 | echo "拉取新代码完成" 30 | 31 | } 32 | 33 | function checkServer() 34 | { 35 | target=`ps -ef | grep "${PROJECT_NAME}" | grep "java" | grep -v "grep"` 36 | if [ -n "${target}" ] ;then 37 | echo "服务正常启动" 38 | echo "${target}" 39 | return 0; 40 | fi 41 | 42 | #这句代码有问题,判断不生效 43 | if [ ! -f "target/${JAR_NAME}" ] ;then 44 | echo "项目执行文件不存在，重新编译" 45 | compile 46 | fi 47 | 48 | echo "start server" 49 | 50 | nohup java -jar ./target/${JAR_NAME} >/dev/null 2>&1 & 51 | 52 | 53 | target=`ps -ef | grep "${PROJECT_NAME}" | grep "java" | grep -v "grep"` 54 | if [ -n "$target" ] ;then 55 | echo "server start success" 56 | echo "$target" 57 | return 0 58 | else 59 | echo "new server start failed please check IT!!!" 60 | return 1 61 | fi 62 | } 63 | 64 | function killServer(){ 65 | target=`ps -ef | grep "${PROJECT_NAME}" | grep "java" | grep -v "grep"` 66 | if [ -n "$target" ] ;then 67 | echo "kill old server process" 68 | ps -ef | grep "${PROJECT_NAME}" | grep -v "grep" | awk '{print $2}' | xargs kill -9 69 | echo `ps -ef | grep "${PROJECT_NAME}" | grep -v "grep"` 70 | 71 | fi 72 | } 73 | 74 | hasCodeUpdate 75 | updateStatus=$? 76 | 77 | if [ ! ${updateStatus} -eq 0 ] ;then 78 | killServer 79 | echo "clean code" 80 | git clean -dfx 81 | echo "pull new code" 82 | git pull 83 | # pullNewCode 84 | compile 85 | fi 86 | checkServer -------------------------------------------------------------------------------- /proxy-kidd/src/main/resources-env/prod/application.yml: -------------------------------------------------------------------------------- 1 | # mysql database 2 | server: 3 | port: 5597 4 | 5 | spring: 6 | datasource: 7 | type: com.alibaba.druid.pool.DruidDataSource 8 | driver-class-name: com.mysql.jdbc.Driver 9 | url: jdbc:mysql://115.28.58.82:3306/phantom?characterEncoding=utf-8&autoReconnect=true&useSSL=false 10 | username: root 11 | password: phantom 12 | filters: stat 13 | maxActive: 20 14 | initialSize: 1 15 | maxWait: 60000 16 | minIdle: 1 17 | timeBetweenEvictionRunsMillis: 60000 18 | minEvictableIdleTimeMillis: 300000 19 | validationQuery: select 1 FROM DUAL 20 | testWhileIdle: true 21 | testOnBorrow: false 22 | testOnReturn: false 23 | poolPreparedStatements: true 24 | maxOpenPreparedStatements: 20 25 | 26 | jpa: 27 | hibernate: 28 | ddl-auto: update 29 | database-platform: org.hibernate.dialect.MySQL5InnoDBDialect 30 | show-sql: true 31 | servlet: 32 | multipart: 33 | max-request-size: 300MB 34 | max-file-size: 1000MB 35 | 36 | #ip_segments ip_list 37 | input_type: ip_segments 38 | 39 | # Ports need to detect 40 | data.address: /root/proxyScanner/ip_data/ 41 | detection.ports.file: /root/proxyScanner/ip_data/DetectPorts.txt 42 | fileter_ip: /root/proxyScanner/ip_data/FilterIPList.txt 43 | 44 | # Scanning frequency configuration 45 | # Start time in 24 hours(0 ~ 23), set -1 to start the scanning immediately 46 | inland.http.scanner.start: -1 47 | inland.socks4.scanner.start: -1 48 | inland.socks5.scanner.start: -1 49 | # Interval in hours, set -1 to close the scanner 50 | inland.http.scanner.interval: 24 51 | inland.socks4.scanner.interval: -1 52 | inland.socks5.scanner.interval: -1 53 | 54 | # Start time in 24 hours(0 ~ 23), set -1 to start the scanning immediately 55 | abroad.http.scanner.start: 15 56 | abroad.socks4.scanner.start: 15 57 | abroad.socks5.scanner.start: 15 58 | # Interval in hours, set -1 to close the scanner 59 | abroad.http.scanner.interval: -1 60 | abroad.socks4.scanner.interval: -1 61 | abroad.socks5.scanner.interval: -1 -------------------------------------------------------------------------------- /proxy-kidd/src/main/resources-env/dev/application.yml: -------------------------------------------------------------------------------- 1 | # mysql database 2 | server: 3 | port: 5597 4 | 5 | spring: 6 | datasource: 7 | type: com.alibaba.druid.pool.DruidDataSource 8 | driver-class-name: com.mysql.jdbc.Driver 9 | url: jdbc:mysql://115.28.58.82:3306/phantom?characterEncoding=utf-8&autoReconnect=true&useSSL=false 10 | username: root 11 | password: phantom 12 | filters: stat 13 | # 最大连接数 14 | maxActive: 20 15 | initialSize: 1 16 | maxWait: 60000 17 | minIdle: 1 18 | timeBetweenEvictionRunsMillis: 60000 19 | minEvictableIdleTimeMillis: 300000 20 | validationQuery: select 1 FROM DUAL 21 | testWhileIdle: true 22 | testOnBorrow: false 23 | testOnReturn: false 24 | poolPreparedStatements: true 25 | maxOpenPreparedStatements: 20 26 | 27 | jpa: 28 | hibernate: 29 | ddl-auto: update 30 | database-platform: org.hibernate.dialect.MySQL5InnoDBDialect 31 | show-sql: true 32 | servlet: 33 | multipart: 34 | max-request-size: 300MB 35 | max-file-size: 1000MB 36 | 37 | #ip_segments ip_list 38 | input_type: ip_segments 39 | 40 | # Ports need to detect 41 | data.address: /Users/xulei2/data/ip_data/ 42 | detection.ports.file: /Users/xulei2/data/ip_data/DetectPorts.txt 43 | fileter_ip: Users/xulei2/data/ip_data/FilterIPList.txt 44 | 45 | # Scanning frequency configuration 46 | # Start time in 24 hours(0 ~ 23), set -1 to start the scanning immediately 47 | inland.http.scanner.start: -1 48 | inland.socks4.scanner.start: -1 49 | inland.socks5.scanner.start: -1 50 | # Interval in hours, set -1 to close the scanner 51 | inland.http.scanner.interval: 24 52 | inland.socks4.scanner.interval: -1 53 | inland.socks5.scanner.interval: -1 54 | 55 | # Start time in 24 hours(0 ~ 23), set -1 to start the scanning immediately 56 | abroad.http.scanner.start: 15 57 | abroad.socks4.scanner.start: 15 58 | abroad.socks5.scanner.start: 15 59 | # Interval in hours, set -1 to close the scanner 60 | abroad.http.scanner.interval: -1 61 | abroad.socks4.scanner.interval: -1 62 | abroad.socks5.scanner.interval: -1 -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/utils/SpringUtil.java: -------------------------------------------------------------------------------- 1 | package com.noisy.utils; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationContextAware; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * @Auther: liuzhaoce 10 | * @Date: 2019-04-29 17:19 11 | * @Description: 12 | */ 13 | 14 | @Component 15 | public class SpringUtil implements ApplicationContextAware { 16 | private static ApplicationContext applicationContext = null; 17 | 18 | @Override 19 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 20 | if(SpringUtil.applicationContext == null){ 21 | SpringUtil.applicationContext = applicationContext; 22 | } 23 | System.out.println("---------------------------------------------------------------------"); 24 | System.out.println("---------------------------------------------------------------------"); 25 | System.out.println("---------------com.kfit.base.util.SpringUtil------------------------------------------------------"); 26 | System.out.println("========ApplicationContext配置成功,在普通类可以通过调用SpringUtils.getAppContext()获取applicationContext对象,applicationContext="+SpringUtil.applicationContext+"========"); 27 | System.out.println("---------------------------------------------------------------------"); 28 | } 29 | 30 | //获取applicationContext 31 | public static ApplicationContext getApplicationContext() { 32 | return applicationContext; 33 | } 34 | 35 | //通过name获取 Bean. 36 | public static Object getBean(String name){ 37 | return getApplicationContext().getBean(name); 38 | } 39 | 40 | //通过class获取Bean. 41 | public static T getBean(Class clazz){ 42 | return getApplicationContext().getBean(clazz); 43 | } 44 | 45 | //通过name,以及Clazz返回指定的Bean 46 | public static T getBean(String name,Class clazz){ 47 | return getApplicationContext().getBean(name, clazz); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/util/IPFilterUtils.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.util; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Created by Lei.x on 5/6/19. 8 | */ 9 | public class IPFilterUtils { 10 | private final static long longMulticastIP = IPSegment.ipToLong("224.0.0.0"); 11 | private final static long classAPrivateIPStart = IPSegment.ipToLong("10.0.0.0"); 12 | private final static long classAPrivateIPEnd = IPSegment.ipToLong("10.255.255.255"); 13 | private final static long classBPrivateIPStart = IPSegment.ipToLong("172.16.0.0"); 14 | private final static long classBPrivateIPEnd = IPSegment.ipToLong("172.31.255.255"); 15 | private final static long classCPrivateIPStart = IPSegment.ipToLong("192.168.0.0"); 16 | private final static long classCPrivateIPEnd = IPSegment.ipToLong("192.168.255.255"); 17 | 18 | private final static IPFilterUtils _Instance = new IPFilterUtils(); 19 | 20 | private Set _FilterIPList = new HashSet(){ 21 | { 22 | add("139.219.135.92"); 23 | add("185.12.64.0"); 24 | add("185.12.64.1"); 25 | add("185.12.64.2"); 26 | add("185.12.64.3"); 27 | add("104.20.65.24"); 28 | 29 | } 30 | }; 31 | 32 | 33 | 34 | private IPFilterUtils() { 35 | 36 | } 37 | 38 | public static IPFilterUtils getInstance() { 39 | return _Instance; 40 | } 41 | 42 | public boolean needFilter(final String ip) { 43 | if (_FilterIPList.contains(ip)) { 44 | return true; 45 | } 46 | 47 | return isPrivateIPOrMulticastIP(ip); 48 | } 49 | 50 | public boolean isPrivateIPOrMulticastIP(final String ip) { 51 | long longIp = IPSegment.ipToLong(ip); 52 | if (longIp >= longMulticastIP) { 53 | return true; 54 | } 55 | 56 | if ((longIp >= classAPrivateIPStart && longIp <= classAPrivateIPEnd) 57 | || (longIp >= classBPrivateIPStart && longIp <= classBPrivateIPEnd) 58 | || (longIp >= classCPrivateIPStart && longIp <= classCPrivateIPEnd)) { 59 | return true; 60 | } 61 | 62 | return false; 63 | } 64 | 65 | } 66 | 67 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/task/Upload5UTask.java: -------------------------------------------------------------------------------- 1 | package com.noisy.task; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.noisy.entity.AgentIp; 5 | import com.noisy.service.impl.AgentIpServiceImpl; 6 | import com.noisy.utils.IPutils; 7 | import com.virjar.dungproxy.client.httpclient.HttpInvoker; 8 | import com.virjar.sipsoup.exception.XpathSyntaxErrorException; 9 | import com.virjar.sipsoup.parse.XpathParser; 10 | import org.jsoup.Jsoup; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.scheduling.annotation.EnableScheduling; 13 | import org.springframework.scheduling.annotation.Scheduled; 14 | import org.springframework.stereotype.Component; 15 | 16 | import javax.annotation.Resource; 17 | import java.time.LocalDateTime; 18 | import java.util.List; 19 | 20 | /** 21 | * @Auther: liuzhaoce 22 | * @Date: 2019-04-30 20:42 23 | * @Description: 24 | */ 25 | 26 | @Component 27 | @Configuration 28 | @EnableScheduling 29 | public class Upload5UTask { 30 | 31 | @Resource 32 | AgentIpServiceImpl agentIpService; 33 | 34 | @Scheduled(fixedRate = 5000) 35 | private void configureTasks() { 36 | 37 | try { 38 | String s = HttpInvoker.get("http://www.data5u.com/free/index.html"); 39 | List msg = XpathParser.compile("//css('.wlist')::ul[position() > 1]/span/li/text()").evaluateToString(Jsoup.parse(s)); 40 | List agentIPs = msgToIP(msg); 41 | System.out.println(agentIPs); 42 | agentIpService.upload(agentIPs); 43 | System.err.println("爬取无忧代理 定时任务时间: " + LocalDateTime.now()); 44 | } catch (XpathSyntaxErrorException e) { 45 | e.printStackTrace(); 46 | } 47 | 48 | } 49 | 50 | private static List msgToIP(List msg) { 51 | 52 | List> listArr = IPutils.createList(msg, 9); 53 | List agentIPList = Lists.newArrayList(); 54 | for (List item : listArr) { 55 | AgentIp agentIp = new AgentIp(item.get(0), item.get(1), item.get(2), item.get(3), item.get(4) + " " + item.get(5), item.get(6)); 56 | agentIPList.add(agentIp); 57 | } 58 | return agentIPList; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /proxy-grab/target/classes/spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /proxy-kidd/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.noisy 8 | phantom 9 | 0.0.1-SNAPSHOT 10 | 11 | com.noisy 12 | proxy-kidd 13 | 1.0-SNAPSHOT 14 | proxy-kidd 15 | http://maven.apache.org 16 | 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | 28 | 29 | 30 | 31 | dev 32 | 33 | dev 34 | 35 | 36 | true 37 | 38 | 39 | 40 | 41 | src/main/resources-env/dev 42 | 43 | 44 | src/main/resources 45 | 46 | 47 | 48 | 49 | 50 | 51 | prod 52 | 53 | prod 54 | 55 | 56 | 57 | 58 | src/main/resources-env/prod 59 | 60 | 61 | src/main/resources 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /proxy-grab/src/main/resources/spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/entity/ProxyInfo.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.entity; 2 | 3 | /** 4 | * Created by kevin on 5/24/16. 5 | */ 6 | public class ProxyInfo { 7 | private String ip; 8 | private int protocol; 9 | private int port; 10 | private int type; 11 | private String location; 12 | private long updateTime; 13 | 14 | public ProxyInfo() { 15 | } 16 | 17 | public ProxyInfo(String ip, int protocol, int port, int type, long updateTime) { 18 | this.ip = ip; 19 | this.protocol = protocol; 20 | this.port = port; 21 | this.type = type; 22 | this.updateTime = updateTime; 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | StringBuilder stringBuilder = new StringBuilder(ip); 28 | stringBuilder.append(","); 29 | stringBuilder.append(protocol); 30 | stringBuilder.append(","); 31 | stringBuilder.append(port); 32 | stringBuilder.append(","); 33 | stringBuilder.append(type); 34 | stringBuilder.append(","); 35 | stringBuilder.append(location); 36 | stringBuilder.append(","); 37 | stringBuilder.append(updateTime); 38 | stringBuilder.append("\n"); 39 | 40 | return stringBuilder.toString(); 41 | } 42 | 43 | public String getIp() { 44 | return ip; 45 | } 46 | 47 | public void setIp(String ip) { 48 | this.ip = ip; 49 | } 50 | 51 | public int getProtocol() { 52 | return protocol; 53 | } 54 | 55 | public void setProtocol(int protocol) { 56 | this.protocol = protocol; 57 | } 58 | 59 | public int getPort() { 60 | return port; 61 | } 62 | 63 | public void setPort(int port) { 64 | this.port = port; 65 | } 66 | 67 | public int getType() { 68 | return type; 69 | } 70 | 71 | public void setType(int type) { 72 | this.type = type; 73 | } 74 | 75 | public String getLocation() { 76 | return location; 77 | } 78 | 79 | public void setLocation(String location) { 80 | this.location = location; 81 | } 82 | 83 | public long getUpdateTime() { 84 | return updateTime; 85 | } 86 | 87 | public void setUpdateTime(long updateTime) { 88 | this.updateTime = updateTime; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/task/UploadKuaidailiTask.java: -------------------------------------------------------------------------------- 1 | package com.noisy.task; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.noisy.entity.AgentIp; 5 | import com.noisy.service.impl.AgentIpServiceImpl; 6 | import com.noisy.utils.IPutils; 7 | import com.virjar.dungproxy.client.httpclient.HttpInvoker; 8 | import com.virjar.sipsoup.exception.XpathSyntaxErrorException; 9 | import com.virjar.sipsoup.parse.XpathParser; 10 | import org.jsoup.Jsoup; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.scheduling.annotation.EnableScheduling; 13 | import org.springframework.stereotype.Component; 14 | 15 | import javax.annotation.Resource; 16 | import java.time.LocalDateTime; 17 | import java.util.List; 18 | 19 | /** 20 | * @Auther: liuzhaoce 21 | * @Date: 2019-05-05 10:48 22 | * @Description: 23 | */ 24 | 25 | @Component 26 | @Configuration 27 | @EnableScheduling 28 | public class UploadKuaidailiTask { 29 | 30 | @Resource 31 | AgentIpServiceImpl agentIpService; 32 | 33 | // @Scheduled(fixedRate = 5000) 34 | private void configureTasks() { 35 | 36 | try { 37 | String s = HttpInvoker.get("https://www.kuaidaili.com/free/"); 38 | List msg = XpathParser.compile("//css('.con-body')::div/div/table/tbody/tr/td/text()").evaluateToString(Jsoup.parse(s)); 39 | List agentIps = msgToIP(msg, 7); 40 | System.out.println(agentIps); 41 | agentIpService.upload(agentIps); 42 | System.err.println("爬取快代理 定时任务时间: " + LocalDateTime.now()); 43 | } catch (XpathSyntaxErrorException e) { 44 | e.printStackTrace(); 45 | } 46 | } 47 | 48 | /** 49 | * 50 | * @Description: msg转换为 List 51 | */ 52 | public static List msgToIP(List msg, int size) { 53 | 54 | List> listArr = IPutils.createList(msg, size); 55 | List agentIPList = Lists.newArrayList(); 56 | for (List item : listArr) { 57 | String[] bits = item.get(4).split(" "); 58 | String operator = bits[bits.length - 1]; 59 | String location = item.get(4).substring(0, item.get(4).indexOf(operator)); 60 | AgentIp agentIp = new AgentIp(item.get(0), item.get(1), item.get(2), item.get(3), location, operator); 61 | agentIPList.add(agentIp); 62 | } 63 | return agentIPList; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/task/UploadXiciTask.java: -------------------------------------------------------------------------------- 1 | package com.noisy.task; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.noisy.entity.AgentIp; 5 | import com.noisy.service.impl.AgentIpServiceImpl; 6 | import com.noisy.utils.IPutils; 7 | import com.virjar.dungproxy.client.httpclient.HttpInvoker; 8 | import com.virjar.sipsoup.exception.XpathSyntaxErrorException; 9 | import com.virjar.sipsoup.parse.XpathParser; 10 | import org.jsoup.Jsoup; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.scheduling.annotation.EnableScheduling; 13 | import org.springframework.stereotype.Component; 14 | 15 | import javax.annotation.Resource; 16 | import java.time.LocalDateTime; 17 | import java.util.List; 18 | 19 | /** 20 | * @Auther: liuzhaoce 21 | * @Date: 2019-05-05 15:38 22 | * @Description: 23 | */ 24 | 25 | @Component 26 | @Configuration 27 | @EnableScheduling 28 | public class UploadXiciTask { 29 | 30 | @Resource 31 | AgentIpServiceImpl agentIpService; 32 | 33 | // @Scheduled(fixedRate = 5000) 34 | private void configureTasks() { 35 | 36 | try { 37 | String s = HttpInvoker.get("https://www.xicidaili.com/nn/1"); 38 | List msg = XpathParser.compile("//css('#ip_list')::tbody/tr[position() > 1]/td/text()").evaluateToString(Jsoup.parse(s)); 39 | List location = XpathParser.compile("//css('#ip_list')::tbody/tr[position() > 1]/td/a/text()").evaluateToString(Jsoup.parse(s)); 40 | System.out.println(msg); 41 | System.out.println(location); 42 | List agentIPs = msgToIP(msg, location); 43 | System.out.println(agentIPs); 44 | agentIpService.upload(agentIPs); 45 | System.err.println("爬取西刺代理 定时任务时间: " + LocalDateTime.now()); 46 | } catch (XpathSyntaxErrorException e) { 47 | e.printStackTrace(); 48 | } 49 | 50 | } 51 | 52 | private static List msgToIP(List msg, List location) { 53 | 54 | List> listArr = IPutils.createList(msg, 10); 55 | List agentIPList = Lists.newArrayList(); 56 | int i = 0; 57 | for (List item : listArr) { 58 | String loc = ""; 59 | if (location.size() > i){ 60 | loc = location.get(i); 61 | } 62 | AgentIp agentIp = new AgentIp(item.get(1), item.get(2), item.get(4), item.get(5), loc, ""); 63 | agentIPList.add(agentIp); 64 | i++; 65 | } 66 | return agentIPList; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/resources-env/dev/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | ${logDir}/${projectName}/${projectName}.log 19 | 20 | ${logDir}/${projectName}/${projectName}-%d{yyyy-MM-dd}.zip 21 | ${maxHistory} 22 | 23 | 24 | ${CONSOLE_LOG_PATTERN} 25 | 26 | 27 | 28 | 29 | 30 | ERROR 31 | ACCEPT 32 | DENY 33 | 34 | ${logDir}/${projectName}/${projectName}-error.log 35 | 36 | ${logDir}/${projectName}/${projectName}-error-%d{yyyy-MM-dd}.zip 37 | ${errorMaxHistory} 38 | 39 | 40 | ${CONSOLE_LOG_PATTERN} 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | ${CONSOLE_LOG_PATTERN} 50 | 51 | 52 | 53 | DEBUG 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/resources-env/prod/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | ${logDir}/${projectName}/${projectName}.log 19 | 20 | ${logDir}/${projectName}/${projectName}-%d{yyyy-MM-dd}.zip 21 | ${maxHistory} 22 | 23 | 24 | ${CONSOLE_LOG_PATTERN} 25 | 26 | 27 | 28 | 29 | 30 | ERROR 31 | ACCEPT 32 | DENY 33 | 34 | ${logDir}/${projectName}/${projectName}-error.log 35 | 36 | ${logDir}/${projectName}/${projectName}-error-%d{yyyy-MM-dd}.zip 37 | ${errorMaxHistory} 38 | 39 | 40 | ${CONSOLE_LOG_PATTERN} 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | ${CONSOLE_LOG_PATTERN} 50 | 51 | 52 | 53 | DEBUG 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/detector/AbstractProxyDetector.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.detector; 2 | 3 | import com.noisy.proxy.entity.ProxyType; 4 | import com.noisy.proxy.util.IPPoolUtils; 5 | import org.apache.commons.lang3.StringUtils; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.annotation.Value; 9 | import org.springframework.stereotype.Component; 10 | 11 | import javax.annotation.Resource; 12 | import java.net.InetAddress; 13 | import java.net.UnknownHostException; 14 | import java.text.SimpleDateFormat; 15 | import java.util.Date; 16 | import java.util.Set; 17 | 18 | /** 19 | * Created by lei.x on 5/24/19. 20 | */ 21 | 22 | public abstract class AbstractProxyDetector implements ProxyDetector { 23 | 24 | 25 | private static final Logger log = LoggerFactory.getLogger(AbstractProxyDetector.class); 26 | //TODO 代理检测URL配置优化 27 | private String proxyCheckerURL = "http://152.136.138.106:20000/test/parse"; 28 | private final int timeout = 10000; 29 | private String outputDir="output/"; 30 | 31 | 32 | public String getProxyCheckerURL() { 33 | return proxyCheckerURL; 34 | } 35 | 36 | public int getTimeout() { 37 | return timeout; 38 | } 39 | 40 | public String getLocalIP() throws UnknownHostException { 41 | 42 | String localIP = null; 43 | try { 44 | localIP = InetAddress.getLocalHost().getHostAddress(); 45 | } catch (UnknownHostException e) { 46 | localIP = null; 47 | log.error("An exception occurred when retrieving the local IP, exception: {}", e); 48 | throw new UnknownHostException("can't get local IP"); 49 | } 50 | 51 | return localIP; 52 | } 53 | 54 | public String getOutputTmpFilePath(String schedulerName) { 55 | if (StringUtils.isEmpty(schedulerName)) { 56 | throw new IllegalArgumentException("The scheduler name cannot be null!"); 57 | } 58 | 59 | StringBuilder filePath = new StringBuilder(outputDir); 60 | filePath.append(schedulerName); 61 | filePath.append("_"); 62 | filePath.append(new SimpleDateFormat("yyyyMMddHHmmss").format(new Date())); 63 | filePath.append(".tmp"); 64 | return filePath.toString(); 65 | } 66 | 67 | public String retrieveOutputFilePath(String outputTmpFilePath) { 68 | String[] tmp = outputTmpFilePath.split("\\."); 69 | return tmp[0] + ".txt"; 70 | } 71 | 72 | public boolean validateIP(final String ip) { 73 | return IPPoolUtils.validateIP(ip); 74 | } 75 | 76 | public ProxyType checkProxyType(String proxyIp, String response) throws UnknownHostException { 77 | 78 | String currentLocalIP = getLocalIP(); 79 | Set responseIPs = IPPoolUtils.retrieveIPFromText(response); 80 | if (responseIPs == null || responseIPs.isEmpty()) { 81 | return null; 82 | } 83 | 84 | if (responseIPs.contains(currentLocalIP)) { 85 | return ProxyType.TRANSPARENT; 86 | } else if (responseIPs.contains(proxyIp)) { 87 | return ProxyType.ANONYMOUS; 88 | } else if (responseIPs.contains("0.0.0.0")) { 89 | return ProxyType.HIGH_ANONYMOUS; 90 | } else { 91 | return ProxyType.DISTORTING; 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/util/IPSegment.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.util; 2 | 3 | 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | /** 9 | * Created by kevin on 5/25/16. 10 | */ 11 | public class IPSegment implements Comparable { 12 | private final static Logger log = LoggerFactory.getLogger(IPSegment.class); 13 | 14 | private Integer randomIndex; 15 | private String ipWithMaskBits; 16 | private String startIP; 17 | private String endIP; 18 | private String netMask; 19 | private long nextIp; 20 | private long startLongIP; 21 | private long endLongIP; 22 | 23 | public IPSegment(String startIP, String endIP, String netMask) { 24 | this.startIP = startIP; 25 | this.startLongIP = ipToLong(startIP); 26 | this.endIP = endIP; 27 | this.netMask = netMask; 28 | this.nextIp = ipToLong(startIP); 29 | this.endLongIP = ipToLong(endIP); 30 | } 31 | 32 | public static long ipToLong(String strIp) { 33 | if (StringUtils.isEmpty(strIp)) { 34 | log.error("The given string IP is empty."); 35 | return -1; 36 | } 37 | 38 | long[] ip = new long[4]; 39 | int position1 = strIp.indexOf("."); 40 | int position2 = strIp.indexOf(".", position1 + 1); 41 | int position3 = strIp.indexOf(".", position2 + 1); 42 | ip[0] = Long.parseLong(strIp.substring(0, position1)); 43 | ip[1] = Long.parseLong(strIp.substring(position1 + 1, position2)); 44 | ip[2] = Long.parseLong(strIp.substring(position2 + 1, position3)); 45 | ip[3] = Long.parseLong(strIp.substring(position3 + 1)); 46 | return (ip[0] << 24) + (ip[1] << 16) + (ip[2] << 8) + ip[3]; 47 | } 48 | 49 | public static String longToIP(long longIp) { 50 | if (longIp < 0) { 51 | log.error("The given long IP is less than 0."); 52 | return "0.0.0.0"; 53 | } 54 | 55 | StringBuffer sb = new StringBuffer(""); 56 | sb.append(String.valueOf((longIp >>> 24))); 57 | sb.append("."); 58 | sb.append(String.valueOf((longIp & 0x00FFFFFF) >>> 16)); 59 | sb.append("."); 60 | sb.append(String.valueOf((longIp & 0x0000FFFF) >>> 8)); 61 | sb.append("."); 62 | sb.append(String.valueOf((longIp & 0x000000FF))); 63 | return sb.toString(); 64 | } 65 | 66 | public String getNextStringIP() { 67 | String nextStringIP = longToIP(nextIp); 68 | nextIp += 1; 69 | return nextStringIP; 70 | } 71 | 72 | public boolean hasNextIP() { 73 | return nextIp <= endLongIP; 74 | } 75 | 76 | /** 77 | * Reset next IP to the start IP. 78 | */ 79 | public void reset() { 80 | this.nextIp = ipToLong(startIP); 81 | } 82 | 83 | public Integer getRandomIndex() { 84 | return randomIndex; 85 | } 86 | 87 | public void setRandomIndex(Integer randomIndex) { 88 | this.randomIndex = randomIndex; 89 | } 90 | 91 | public String getIpWithMaskBits() { 92 | return ipWithMaskBits; 93 | } 94 | 95 | public void setIpWithMaskBits(String ipWithMaskBits) { 96 | this.ipWithMaskBits = ipWithMaskBits; 97 | } 98 | 99 | public String getStartIP() { 100 | return startIP; 101 | } 102 | 103 | public void setStartIP(String startIP) { 104 | this.startIP = startIP; 105 | nextIp = ipToLong(startIP); 106 | } 107 | 108 | public String getEndIP() { 109 | return endIP; 110 | } 111 | 112 | public void setEndIP(String endIP) { 113 | this.endIP = endIP; 114 | this.endLongIP = ipToLong(endIP); 115 | } 116 | 117 | public String getNetMask() { 118 | return netMask; 119 | } 120 | 121 | public void setNetMask(String netMask) { 122 | this.netMask = netMask; 123 | } 124 | 125 | public long getStartLongIP() { 126 | return startLongIP; 127 | } 128 | 129 | public void setStartLongIP(long startLongIP) { 130 | this.startLongIP = startLongIP; 131 | } 132 | 133 | public long getEndLongIP() { 134 | return endLongIP; 135 | } 136 | 137 | public void setEndLongIP(long endLongIP) { 138 | this.endLongIP = endLongIP; 139 | } 140 | 141 | @Override 142 | public int compareTo(IPSegment o) { 143 | return this.getRandomIndex().compareTo(o.getRandomIndex()); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/EmailReporter.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy; 2 | 3 | import org.apache.commons.mail.DefaultAuthenticator; 4 | import org.apache.commons.mail.Email; 5 | import org.apache.commons.mail.EmailException; 6 | import org.apache.commons.mail.SimpleEmail; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.net.InetAddress; 11 | import java.text.SimpleDateFormat; 12 | import java.util.Date; 13 | 14 | /** 15 | * Created by kevin on 5/31/16. 16 | */ 17 | public class EmailReporter { 18 | private final static Logger log = LoggerFactory.getLogger(EmailReporter.class); 19 | 20 | //TODO 转移到application.properties 21 | // email 配置 22 | private String emailHostName = "smtp.office365.com"; 23 | private int emailSmtpPort = 587; 24 | private String emailAuthUsername = "kaiping.he@maxent-inc.com"; 25 | private String emailAuthPassword = "Maxent-khe"; 26 | private String emailtoList = "381827702@qq.com"; 27 | 28 | 29 | private final static EmailReporter _instance = new EmailReporter(); 30 | private final static long dayMilliseconds = 24 * 60 * 60 * 1000; 31 | private final static long hourMilliseconds = 60 * 60 * 1000; 32 | private final static long minuteMilliseconds = 60 * 1000; 33 | 34 | private final Email email = new SimpleEmail(); 35 | 36 | public static EmailReporter getInstance() { 37 | return _instance; 38 | } 39 | 40 | private EmailReporter() { 41 | try { 42 | init(); 43 | } catch (EmailException e) { 44 | log.error("Faild to initialize the email accounts info."); 45 | } 46 | } 47 | 48 | private void init() throws EmailException { 49 | email.setHostName(emailHostName); 50 | email.setSmtpPort(emailSmtpPort); 51 | email.setAuthenticator(new DefaultAuthenticator( 52 | emailAuthUsername, emailAuthPassword)); 53 | email.setStartTLSEnabled(true); 54 | email.setFrom(emailAuthUsername); 55 | String[] toList = emailtoList.split("\\|"); 56 | for (String to : toList) { 57 | email.addTo(to); 58 | } 59 | 60 | email.setSubject("代理服务器IP扫描报告"); 61 | } 62 | 63 | public void report(String scanTarget, Date startTime, long totalIPNum, int portNum, long proxyIPNum, long consumedTime) { 64 | String startTimeStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(startTime); 65 | 66 | StringBuilder consumedTimeStr = new StringBuilder(); 67 | if (consumedTime >= dayMilliseconds) { 68 | long days = consumedTime / dayMilliseconds; 69 | consumedTimeStr.append(days); 70 | consumedTimeStr.append("天"); 71 | } 72 | 73 | if (consumedTime >= hourMilliseconds) { 74 | long hours = (consumedTime % dayMilliseconds) / hourMilliseconds; 75 | consumedTimeStr.append(hours); 76 | consumedTimeStr.append("小时"); 77 | } 78 | 79 | if (consumedTime >= minuteMilliseconds) { 80 | long minutes = (consumedTime % hourMilliseconds) / minuteMilliseconds; 81 | consumedTimeStr.append(minutes); 82 | consumedTimeStr.append("分钟"); 83 | } 84 | 85 | long seconds = (consumedTime % minuteMilliseconds) / 1000; 86 | consumedTimeStr.append(seconds); 87 | consumedTimeStr.append("秒"); 88 | 89 | String hostname = "localhost"; 90 | try { 91 | hostname = InetAddress.getLocalHost().getHostName(); 92 | } catch (Exception e) { 93 | log.error("Faild to retrieve the host name of the this server, ex: {}", e); 94 | } 95 | long speed = (totalIPNum * portNum * 1000) / consumedTime; 96 | 97 | StringBuilder msgBuilder = new StringBuilder("扫描服务主机："); 98 | msgBuilder.append(hostname); 99 | msgBuilder.append("\n扫描目标："); 100 | msgBuilder.append(scanTarget); 101 | msgBuilder.append("\n开始时间："); 102 | msgBuilder.append(startTimeStr); 103 | msgBuilder.append("\n扫描IP总数："); 104 | msgBuilder.append(totalIPNum); 105 | msgBuilder.append("\n扫描端口数："); 106 | msgBuilder.append(portNum); 107 | msgBuilder.append("\n代理IP数："); 108 | msgBuilder.append(proxyIPNum); 109 | msgBuilder.append("\n探测速度："); 110 | msgBuilder.append(speed); 111 | msgBuilder.append(" 次/秒"); 112 | msgBuilder.append("\n共计耗时："); 113 | msgBuilder.append(consumedTimeStr); 114 | 115 | try { 116 | email.setMsg(msgBuilder.toString()); 117 | email.send(); 118 | } catch (EmailException e) { 119 | log.error("Failed to send email for report the scanning results, ex: {}", e); 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | https://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | 20 | import java.io.File; 21 | import java.io.FileInputStream; 22 | import java.io.FileOutputStream; 23 | import java.io.IOException; 24 | import java.net.URL; 25 | import java.nio.channels.Channels; 26 | import java.nio.channels.ReadableByteChannel; 27 | import java.util.Properties; 28 | 29 | public class MavenWrapperDownloader { 30 | 31 | /** 32 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 33 | */ 34 | private static final String DEFAULT_DOWNLOAD_URL = 35 | "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"; 36 | 37 | /** 38 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 39 | * use instead of the default one. 40 | */ 41 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 42 | ".mvn/wrapper/maven-wrapper.properties"; 43 | 44 | /** 45 | * Path where the maven-wrapper.jar will be saved to. 46 | */ 47 | private static final String MAVEN_WRAPPER_JAR_PATH = 48 | ".mvn/wrapper/maven-wrapper.jar"; 49 | 50 | /** 51 | * Name of the property which should be used to override the default download url for the wrapper. 52 | */ 53 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 54 | 55 | public static void main(String args[]) { 56 | System.out.println("- Downloader started"); 57 | File baseDirectory = new File(args[0]); 58 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 59 | 60 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 61 | // wrapperUrl parameter. 62 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 63 | String url = DEFAULT_DOWNLOAD_URL; 64 | if(mavenWrapperPropertyFile.exists()) { 65 | FileInputStream mavenWrapperPropertyFileInputStream = null; 66 | try { 67 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 68 | Properties mavenWrapperProperties = new Properties(); 69 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 70 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 71 | } catch (IOException e) { 72 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 73 | } finally { 74 | try { 75 | if(mavenWrapperPropertyFileInputStream != null) { 76 | mavenWrapperPropertyFileInputStream.close(); 77 | } 78 | } catch (IOException e) { 79 | // Ignore ... 80 | } 81 | } 82 | } 83 | System.out.println("- Downloading from: : " + url); 84 | 85 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 86 | if(!outputFile.getParentFile().exists()) { 87 | if(!outputFile.getParentFile().mkdirs()) { 88 | System.out.println( 89 | "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 90 | } 91 | } 92 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 93 | try { 94 | downloadFileFromURL(url, outputFile); 95 | System.out.println("Done"); 96 | System.exit(0); 97 | } catch (Throwable e) { 98 | System.out.println("- Error downloading"); 99 | e.printStackTrace(); 100 | System.exit(1); 101 | } 102 | } 103 | 104 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 105 | URL website = new URL(urlString); 106 | ReadableByteChannel rbc; 107 | rbc = Channels.newChannel(website.openStream()); 108 | FileOutputStream fos = new FileOutputStream(destination); 109 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 110 | fos.close(); 111 | rbc.close(); 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /proxy-grab/src/main/java/com/noisy/utils/CodeGenerator.java: -------------------------------------------------------------------------------- 1 | package com.noisy.utils; 2 | 3 | import com.baomidou.mybatisplus.core.exceptions.MybatisPlusException; 4 | import com.baomidou.mybatisplus.core.toolkit.StringPool; 5 | import com.baomidou.mybatisplus.generator.AutoGenerator; 6 | import com.baomidou.mybatisplus.generator.InjectionConfig; 7 | import com.baomidou.mybatisplus.generator.config.*; 8 | import com.baomidou.mybatisplus.generator.config.po.TableInfo; 9 | import com.baomidou.mybatisplus.generator.config.rules.NamingStrategy; 10 | import com.baomidou.mybatisplus.generator.engine.FreemarkerTemplateEngine; 11 | import org.apache.commons.lang3.StringUtils; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.Scanner; 16 | 17 | /** 18 | * @Auther: liuzhaoce 19 | * @Date: 2019-04-29 10:44 20 | * @Description: 21 | */ 22 | // 演示例子，执行 main 方法控制台输入模块表名回车自动生成对应项目目录中 23 | public class CodeGenerator { 24 | 25 | /** 26 | *

27 | * 读取控制台内容 28 | *

29 | */ 30 | public static String scanner(String tip) { 31 | Scanner scanner = new Scanner(System.in); 32 | StringBuilder help = new StringBuilder(); 33 | help.append("请输入" + tip + "："); 34 | System.out.println(help.toString()); 35 | if (scanner.hasNext()) { 36 | String ipt = scanner.next(); 37 | if (StringUtils.isNotEmpty(ipt)) { 38 | return ipt; 39 | } 40 | } 41 | throw new MybatisPlusException("请输入正确的" + tip + "！"); 42 | } 43 | 44 | public static void main(String[] args) { 45 | // 代码生成器 46 | AutoGenerator mpg = new AutoGenerator(); 47 | 48 | // 全局配置 49 | GlobalConfig gc = new GlobalConfig(); 50 | String projectPath = System.getProperty("user.dir"); 51 | gc.setOutputDir(projectPath + "/proxy-grab/src/main/java"); 52 | gc.setAuthor("liuzhaoce"); 53 | gc.setOpen(false); 54 | // gc.setSwagger2(true); 实体属性 Swagger2 注解 55 | mpg.setGlobalConfig(gc); 56 | 57 | // 数据源配置 58 | DataSourceConfig dsc = new DataSourceConfig(); 59 | dsc.setUrl("jdbc:mysql://localhost:3306/proxy_grab?characterEncoding=utf-8&autoReconnect=true&useSSL=false"); 60 | // dsc.setSchemaName("public"); 61 | dsc.setDriverName("com.mysql.cj.jdbc.Driver"); 62 | dsc.setUsername("root"); 63 | dsc.setPassword("2244363365lzhce&"); 64 | mpg.setDataSource(dsc); 65 | 66 | // 包配置 67 | PackageConfig pc = new PackageConfig(); 68 | // pc.setModuleName(scanner("模块名")); 69 | pc.setParent("com.noisy"); 70 | mpg.setPackageInfo(pc); 71 | 72 | // 自定义配置 73 | InjectionConfig cfg = new InjectionConfig() { 74 | @Override 75 | public void initMap() { 76 | // to do nothing 77 | } 78 | }; 79 | 80 | // 如果模板引擎是 freemarker 81 | String templatePath = "/templates/mapper.xml.ftl"; 82 | // 如果模板引擎是 velocity 83 | // String templatePath = "/templates/mapper.xml.vm"; 84 | 85 | // 自定义输出配置 86 | List focList = new ArrayList<>(); 87 | // 自定义配置会被优先输出 88 | focList.add(new FileOutConfig(templatePath) { 89 | @Override 90 | public String outputFile(TableInfo tableInfo) { 91 | // 自定义输出文件名 ， 如果你 Entity 设置了前后缀、此处注意 xml 的名称会跟着发生变化！！ 92 | return projectPath + "/proxy-grab/src/main/resources/mapper/" 93 | + "/" + tableInfo.getEntityName() + "Mapper" + StringPool.DOT_XML; 94 | } 95 | }); 96 | /* 97 | cfg.setFileCreate(new IFileCreate() { 98 | @Override 99 | public boolean isCreate(ConfigBuilder configBuilder, FileType fileType, String filePath) { 100 | // 判断自定义文件夹是否需要创建 101 | checkDir("调用默认方法创建的目录"); 102 | return false; 103 | } 104 | }); 105 | */ 106 | cfg.setFileOutConfigList(focList); 107 | mpg.setCfg(cfg); 108 | 109 | // 配置模板 110 | TemplateConfig templateConfig = new TemplateConfig(); 111 | 112 | // 配置自定义输出模板 113 | //指定自定义模板路径，注意不要带上.ftl/.vm, 会根据使用的模板引擎自动识别 114 | // templateConfig.setEntity("templates/entity2.java"); 115 | // templateConfig.setService(); 116 | // templateConfig.setController(); 117 | 118 | templateConfig.setXml(null); 119 | mpg.setTemplate(templateConfig); 120 | 121 | // 策略配置 122 | StrategyConfig strategy = new StrategyConfig(); 123 | strategy.setNaming(NamingStrategy.underline_to_camel); 124 | strategy.setColumnNaming(NamingStrategy.underline_to_camel); 125 | strategy.setEntityLombokModel(true); 126 | strategy.setRestControllerStyle(true); 127 | strategy.setInclude(scanner("表名，多个英文逗号分割").split(",")); 128 | strategy.setSuperEntityColumns("id"); 129 | strategy.setControllerMappingHyphenStyle(true); 130 | strategy.setTablePrefix(pc.getModuleName() + "_"); 131 | mpg.setStrategy(strategy); 132 | mpg.setTemplateEngine(new FreemarkerTemplateEngine()); 133 | mpg.execute(); 134 | } 135 | 136 | } 137 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | pom 5 | 6 | proxy-kidd 7 | phantom-backend 8 | proxy-grab 9 | phantom-master 10 | 11 | 12 | org.springframework.boot 13 | spring-boot-starter-parent 14 | 2.1.4.RELEASE 15 | 16 | 17 | com.noisy 18 | phantom 19 | 0.0.1-SNAPSHOT 20 | phantom 21 | Demo project for Spring Boot 22 | 23 | 24 | 1.8 25 | 26 | 27 | 28 | 29 | 30 | com.noisy 31 | proxy-kidd 32 | ${project.version} 33 | 34 | 35 | com.noisy 36 | phantom-backend 37 | ${project.version} 38 | 39 | 40 | com.noisy 41 | proxy-grab 42 | ${project.version} 43 | 44 | 45 | 46 | 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-starter-data-jpa 51 | 52 | 53 | org.springframework.boot 54 | spring-boot-starter 55 | 56 | 57 | 58 | 59 | com.baomidou 60 | mybatis-plus 61 | 3.1.1 62 | 63 | 64 | 65 | org.springframework 66 | spring-web 67 | 5.1.6.RELEASE 68 | 69 | 70 | 71 | com.baomidou 72 | mybatis-plus-boot-starter 73 | 3.1.1 74 | 75 | 76 | junit 77 | junit 78 | 4.10 79 | 80 | 81 | 82 | org.springframework.boot 83 | spring-boot-test 84 | 2.0.3.RELEASE 85 | 86 | 87 | 88 | 89 | org.springframework.boot 90 | spring-boot-starter-web 91 | 92 | 93 | org.projectlombok 94 | lombok 95 | true 96 | 97 | 98 | org.springframework.boot 99 | spring-boot-starter-test 100 | test 101 | 102 | 103 | commons-configuration 104 | commons-configuration 105 | 1.9 106 | 107 | 108 | commons-io 109 | commons-io 110 | 2.6 111 | 112 | 113 | org.apache.commons 114 | commons-lang3 115 | 3.8.1 116 | 117 | 118 | 119 | io.netty 120 | netty-all 121 | 4.1.35.Final 122 | 123 | 124 | org.apache.commons 125 | commons-email 126 | 1.4 127 | 128 | 129 | us.monoid.web 130 | resty 131 | 0.3.2 132 | 133 | 134 | cn.wanghaomiao 135 | SeimiCrawler 136 | 2.1.1 137 | 138 | 139 | 140 | com.virjar 141 | sipsoup 142 | RELEASE 143 | 144 | 145 | 146 | com.baomidou 147 | mybatis-plus-generator 148 | 3.1.1 149 | 150 | 151 | 152 | 153 | org.freemarker 154 | freemarker 155 | 2.3.28 156 | 157 | 158 | 159 | 160 | com.alibaba 161 | druid 162 | 1.1.10 163 | 164 | 165 | 166 | 167 | mysql 168 | mysql-connector-java 169 | 5.1.40 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | org.springframework.boot 178 | spring-boot-maven-plugin 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" 124 | FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( 125 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 126 | ) 127 | 128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 130 | if exist %WRAPPER_JAR% ( 131 | echo Found %WRAPPER_JAR% 132 | ) else ( 133 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 134 | echo Downloading from: %DOWNLOAD_URL% 135 | powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" 136 | echo Finished downloading %WRAPPER_JAR% 137 | ) 138 | @REM End of extension 139 | 140 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 141 | if ERRORLEVEL 1 goto error 142 | goto end 143 | 144 | :error 145 | set ERROR_CODE=1 146 | 147 | :end 148 | @endlocal & set ERROR_CODE=%ERROR_CODE% 149 | 150 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 151 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 152 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 153 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 154 | :skipRcPost 155 | 156 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 157 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 158 | 159 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 160 | 161 | exit /B %ERROR_CODE% 162 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/TaskScheduler.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy; 2 | 3 | import com.noisy.proxy.detector.ProxyDetector; 4 | import com.noisy.proxy.detector.ProxyDetectorFactory; 5 | import com.noisy.proxy.entity.InputType; 6 | import com.noisy.proxy.entity.ProtocolType; 7 | import com.noisy.proxy.util.IPFilterUtils; 8 | import com.noisy.proxy.util.IPPoolUtils; 9 | import com.noisy.proxy.util.IPSegment; 10 | import org.apache.commons.lang3.StringUtils; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Iterator; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.Set; 18 | 19 | /** 20 | * Created by kevin on 5/25/16. 21 | */ 22 | 23 | public class TaskScheduler implements Runnable { 24 | 25 | 26 | private final static Logger log = LoggerFactory.getLogger(TaskScheduler.class); 27 | 28 | private Set ports; 29 | private String name; 30 | private String scanTarget; 31 | private Map ipSegments; 32 | private Set ipList; 33 | private ProtocolType protocolType; 34 | private ProxyDetector proxyDetector; 35 | private long totalTasks; 36 | private String inputType; 37 | 38 | public TaskScheduler(String name, String scanTarget, ProtocolType protocolType, Map ipSegments,Set ports) { 39 | if (StringUtils.isEmpty(name) || StringUtils.isEmpty(scanTarget) 40 | || protocolType == null || ipSegments == null) { 41 | throw new IllegalArgumentException( 42 | "Any one of the parameters cannot be null when constructing the TaskScheduler."); 43 | } 44 | log.info("Building scheduler: {}", name); 45 | this.name = name; 46 | this.scanTarget = scanTarget; 47 | this.protocolType = protocolType; 48 | this.ipSegments = ipSegments; 49 | this.ports = ports; 50 | 51 | proxyDetector = ProxyDetectorFactory.createProxyDetector(protocolType, this); 52 | totalTasks = 0; 53 | inputType = InputType.IP_SEGMENTS.getType(); 54 | } 55 | 56 | public TaskScheduler(String name, String scanTarget, ProtocolType protocolType, Set ipList,Set ports) { 57 | if (StringUtils.isEmpty(name) || StringUtils.isEmpty(scanTarget) 58 | || protocolType == null || ipList == null) { 59 | throw new IllegalArgumentException( 60 | "Any one of the parameters cannot be null when constructing the TaskScheduler."); 61 | } 62 | log.info("Building scheduler: {}", name); 63 | this.name = name; 64 | this.scanTarget = scanTarget; 65 | this.protocolType = protocolType; 66 | this.ipList = ipList; 67 | this.ports = ports; 68 | 69 | proxyDetector = ProxyDetectorFactory.createProxyDetector(protocolType, this); 70 | totalTasks = 0; 71 | inputType = InputType.IP_LIST.getType(); 72 | } 73 | 74 | @Override 75 | public void run() { 76 | proxyDetector.reset(); 77 | log.info("The scheduler {} started to execute the new schedule with {} input.", name, inputType); 78 | if (InputType.IP_LIST.isThisType(inputType)) { 79 | createTaskFromIpList(); 80 | } else { 81 | createTaskFromIpSegs(); 82 | } 83 | } 84 | 85 | private void createTaskFromIpSegs() { 86 | if (ipSegments == null || ipSegments.isEmpty()) { 87 | return ; 88 | } 89 | //TODO 计算量太大 90 | // countTotalTasks(); 91 | 92 | log.info("{}: creating tasks.", name); 93 | for (Integer port : ports) { 94 | List ipSegmentList = IPPoolUtils.getIPSegments(ipSegments.values()); 95 | while (!ipSegmentList.isEmpty()) { 96 | Iterator iterator = ipSegmentList.iterator(); 97 | while (iterator.hasNext()) { 98 | IPSegment ipSegment = iterator.next(); 99 | if (ipSegment.hasNextIP()) { 100 | String proxyIP = ipSegment.getNextStringIP(); 101 | if (!IPFilterUtils.getInstance().needFilter(proxyIP)) { 102 | proxyDetector.detect(proxyIP, port); 103 | log.info("proxy detect start...proxyIP:{} ### port:{}",proxyIP,port); 104 | } 105 | } else { 106 | iterator.remove(); 107 | } 108 | } 109 | } 110 | } 111 | 112 | // long totalIPNum = totalTasks / ports.size(); 113 | // log.info("{} has finished to create the tasks, total IP number: {}", name, totalIPNum); 114 | 115 | } 116 | 117 | private long createTaskFromIpList() { 118 | countTotalTasks(); 119 | 120 | log.info("{}: creating tasks.", name); 121 | for (Integer port : ports) { 122 | for (String proxyIP : ipList) { 123 | 124 | proxyDetector.detect(proxyIP, port); 125 | log.info("proxy detect start...proxyIP:{} ### port:{}",proxyIP,port); 126 | } 127 | } 128 | 129 | long totalIPNum = totalTasks / ports.size(); 130 | log.info("{} has finished to create the tasks, total IP number: {}", name, totalIPNum); 131 | 132 | return totalTasks; 133 | } 134 | 135 | public long countTotalTasks() { 136 | if (totalTasks > 0) { 137 | return totalTasks; 138 | } 139 | 140 | log.info("Counting tasks."); 141 | if (InputType.IP_LIST.isThisType(inputType)) { 142 | totalTasks = ports.size() * (ipList == null ? 0 : ipList.size()); 143 | } else { 144 | for (int i = 0; i < ports.size(); i++) { 145 | List ipSegmentList = IPPoolUtils.getIPSegments(ipSegments.values()); 146 | while (!ipSegmentList.isEmpty()) { 147 | Iterator iterator = ipSegmentList.iterator(); 148 | while (iterator.hasNext()) { 149 | IPSegment ipSegment = iterator.next(); 150 | if (ipSegment.hasNextIP()) { 151 | String proxyIP = ipSegment.getNextStringIP(); 152 | if (!IPFilterUtils.getInstance().needFilter(proxyIP)) { 153 | totalTasks++; 154 | } 155 | } else { 156 | iterator.remove(); 157 | } 158 | } 159 | } 160 | } 161 | } 162 | log.info("Total tasks: {}", totalTasks); 163 | 164 | return totalTasks; 165 | } 166 | 167 | public String getName() { 168 | return name; 169 | } 170 | 171 | public String getScanTarget() { 172 | return scanTarget; 173 | } 174 | 175 | public void setScanTarget(String scanTarget) { 176 | this.scanTarget = scanTarget; 177 | } 178 | 179 | public Map getIpSegments() { 180 | return ipSegments; 181 | } 182 | 183 | public ProtocolType getProtocolType() { 184 | return protocolType; 185 | } 186 | 187 | public void setProtocolType(ProtocolType protocolType) { 188 | this.protocolType = protocolType; 189 | } 190 | 191 | public int getScanPortNum() { 192 | return ports.size(); 193 | } 194 | 195 | public ProxyDetector getProxyDetector() { 196 | return proxyDetector; 197 | } 198 | 199 | public long getTotalTasks() { 200 | return totalTasks; 201 | } 202 | 203 | public void setTotalTasks(long totalTasks) { 204 | this.totalTasks = totalTasks; 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | ########################################################################################## 204 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 205 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 206 | ########################################################################################## 207 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 208 | if [ "$MVNW_VERBOSE" = true ]; then 209 | echo "Found .mvn/wrapper/maven-wrapper.jar" 210 | fi 211 | else 212 | if [ "$MVNW_VERBOSE" = true ]; then 213 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 214 | fi 215 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" 216 | while IFS="=" read key value; do 217 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 218 | esac 219 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 220 | if [ "$MVNW_VERBOSE" = true ]; then 221 | echo "Downloading from: $jarUrl" 222 | fi 223 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 224 | 225 | if command -v wget > /dev/null; then 226 | if [ "$MVNW_VERBOSE" = true ]; then 227 | echo "Found wget ... using wget" 228 | fi 229 | wget "$jarUrl" -O "$wrapperJarPath" 230 | elif command -v curl > /dev/null; then 231 | if [ "$MVNW_VERBOSE" = true ]; then 232 | echo "Found curl ... using curl" 233 | fi 234 | curl -o "$wrapperJarPath" "$jarUrl" 235 | else 236 | if [ "$MVNW_VERBOSE" = true ]; then 237 | echo "Falling back to using Java to download" 238 | fi 239 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 240 | if [ -e "$javaClass" ]; then 241 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 242 | if [ "$MVNW_VERBOSE" = true ]; then 243 | echo " - Compiling MavenWrapperDownloader.java ..." 244 | fi 245 | # Compiling the Java class 246 | ("$JAVA_HOME/bin/javac" "$javaClass") 247 | fi 248 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 249 | # Running the downloader 250 | if [ "$MVNW_VERBOSE" = true ]; then 251 | echo " - Running MavenWrapperDownloader.java ..." 252 | fi 253 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 254 | fi 255 | fi 256 | fi 257 | fi 258 | ########################################################################################## 259 | # End of extension 260 | ########################################################################################## 261 | 262 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 263 | if [ "$MVNW_VERBOSE" = true ]; then 264 | echo $MAVEN_PROJECTBASEDIR 265 | fi 266 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 267 | 268 | # For Cygwin, switch paths to Windows format before running java 269 | if $cygwin; then 270 | [ -n "$M2_HOME" ] && 271 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 272 | [ -n "$JAVA_HOME" ] && 273 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 274 | [ -n "$CLASSPATH" ] && 275 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 276 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 277 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 278 | fi 279 | 280 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 281 | 282 | exec "$JAVACMD" \ 283 | $MAVEN_OPTS \ 284 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 285 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 286 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 287 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/detector/HTTPProxyDetector.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.detector; 2 | 3 | import com.noisy.proxy.TaskScheduler; 4 | import com.noisy.proxy.dao.ProxyInfoDao; 5 | import com.noisy.proxy.dao.ProxyInfoDaoFilempl; 6 | import com.noisy.proxy.entity.ProtocolType; 7 | import com.noisy.proxy.entity.ProxyInfo; 8 | import com.noisy.proxy.entity.ProxyType; 9 | import io.netty.bootstrap.Bootstrap; 10 | import io.netty.buffer.ByteBuf; 11 | import io.netty.channel.*; 12 | import io.netty.channel.nio.NioEventLoopGroup; 13 | import io.netty.channel.socket.nio.NioSocketChannel; 14 | import io.netty.handler.codec.http.*; 15 | import io.netty.handler.timeout.ReadTimeoutHandler; 16 | import io.netty.handler.timeout.WriteTimeoutHandler; 17 | import io.netty.util.AttributeKey; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.io.File; 22 | import java.net.URI; 23 | import java.util.Calendar; 24 | import java.util.concurrent.Semaphore; 25 | import java.util.concurrent.TimeUnit; 26 | import java.util.concurrent.atomic.AtomicLong; 27 | 28 | /** 29 | * Created by lei.x on 5/24/19. 30 | */ 31 | public class HTTPProxyDetector extends AbstractProxyDetector { 32 | private static final Logger log = LoggerFactory.getLogger(HTTPProxyDetector.class); 33 | // 线程池核心数 Config the thread pool size by the multiple of the CPU cores, must small than 20 34 | private final static int THREAD_POOL_SIZE_CORES_MULTIPLE = 2; 35 | private final static int nThreads = THREAD_POOL_SIZE_CORES_MULTIPLE 36 | * Runtime.getRuntime().availableProcessors(); 37 | 38 | private final ProxyInfoDao proxyInfoDao = new ProxyInfoDaoFilempl(); 39 | 40 | private EventLoopGroup workerGroup; 41 | private Calendar startTime = Calendar.getInstance(); 42 | private final AtomicLong taskCounter = new AtomicLong(0); 43 | private final AtomicLong totalTasks = new AtomicLong(0); 44 | private final AtomicLong proxyIPNum = new AtomicLong(0); 45 | // The TCP connection limits of the system, must small than 500000 46 | private final int tcpConnLimits = 32000; 47 | private final Semaphore semaphore = new Semaphore(tcpConnLimits); 48 | private final Semaphore canStartNextSchSem = new Semaphore(1); 49 | private Bootstrap bootstrap; 50 | private String outputTmpFilePath; 51 | private File outputTmpFile; 52 | private TaskScheduler scheduler; 53 | 54 | public HTTPProxyDetector(TaskScheduler scheduler) { 55 | this.scheduler = scheduler; 56 | workerGroup = new NioEventLoopGroup(nThreads); 57 | 58 | bootstrap = new Bootstrap(); 59 | bootstrap.group(workerGroup); 60 | bootstrap.channel(NioSocketChannel.class); 61 | bootstrap.option(ChannelOption.AUTO_CLOSE, true); 62 | bootstrap.option(ChannelOption.SO_REUSEADDR, true); 63 | bootstrap.option(ChannelOption.TCP_NODELAY, true); 64 | bootstrap.option(ChannelOption.SO_KEEPALIVE, false); 65 | bootstrap.option(ChannelOption.SO_SNDBUF, 1024); 66 | bootstrap.option(ChannelOption.SO_RCVBUF, 4096); 67 | bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator()); 68 | bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getTimeout()); 69 | 70 | 71 | bootstrap.handler(new ChannelInitializer() { 72 | @Override 73 | public void initChannel(NioSocketChannel ch) throws Exception { 74 | // 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码 75 | ch.pipeline().addLast(new HttpClientCodec()); 76 | ch.pipeline().addLast(new ReadTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 77 | ch.pipeline().addLast(new HttpClientOutboundHandler()); 78 | ch.pipeline().addLast(new WriteTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 79 | // 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码 80 | // ch.pipeline().addLast(new HttpRequestEncoder()); 81 | 82 | ch.pipeline().addLast(new HTTPProxyRespHandler()); 83 | } 84 | }); 85 | 86 | outputTmpFilePath = getOutputTmpFilePath(scheduler.getName()); 87 | outputTmpFile = new File(outputTmpFilePath); 88 | if (!outputTmpFile.getParentFile().exists()) { 89 | outputTmpFile.getParentFile().mkdirs(); 90 | } 91 | } 92 | 93 | public void reset() { 94 | try { 95 | canStartNextSchSem.acquire(); 96 | } catch (InterruptedException e) { 97 | log.error("An exception occurred when resetting the HTTP proxy detector, ex: {}", e); 98 | } 99 | startTime.setTimeInMillis(System.currentTimeMillis()); 100 | totalTasks.set(taskCounter.get()); 101 | proxyIPNum.set(0); 102 | outputTmpFilePath = getOutputTmpFilePath(scheduler.getName()); 103 | outputTmpFile = new File(outputTmpFilePath); 104 | if (!outputTmpFile.getParentFile().exists()) { 105 | outputTmpFile.getParentFile().mkdirs(); 106 | } 107 | } 108 | 109 | @Override 110 | public void detect(String ip, int port) { 111 | try { 112 | semaphore.acquire(); 113 | log.info("semaphore acquire:{}", semaphore.toString()); 114 | } catch (InterruptedException e) { 115 | log.warn("An exception occurred when creating a detection task, ex: {}", e); 116 | } 117 | 118 | // Start the client. 119 | try { 120 | ChannelFuture f = bootstrap.connect(ip, port); 121 | f.channel().attr(AttributeKey.valueOf("ip")).set(ip); 122 | f.channel().attr(AttributeKey.valueOf("port")).set(port); 123 | 124 | long leftTasks = taskCounter.get(); 125 | long tmpTotalTasks = totalTasks.incrementAndGet(); 126 | 127 | 128 | // if (((tmpTotalTasks == scheduler.getTotalTasks()) && (leftTasks % 100) == 0) 129 | // || ((tmpTotalTasks % 1000000) == 0)) { 130 | // log.info("Created tasks: {}, Left tasks: {}", tmpTotalTasks, leftTasks); 131 | // } 132 | } catch (Exception e) { 133 | log.error("An exception occurred when connecting to host {}:{}, ex: {}", ip, port, e); 134 | } 135 | } 136 | 137 | private synchronized void finish(ChannelHandlerContext ctx) throws InterruptedException { 138 | // 避免重复调用 139 | if (ctx.channel().hasAttr(AttributeKey.valueOf("finished"))) { 140 | return; 141 | } 142 | 143 | semaphore.release(); 144 | log.info("semaphore release:{}", semaphore.toString()); 145 | ctx.channel().attr(AttributeKey.valueOf("finished")).set(true); 146 | 147 | long leftTasks = taskCounter.decrementAndGet(); 148 | if (leftTasks < 1) { 149 | log.info("Total tasks: {}, Left tasks: {}", totalTasks, taskCounter); 150 | canStartNextSchSem.release(); 151 | 152 | long consumedTime = System.currentTimeMillis() - startTime.getTimeInMillis(); 153 | // EmailReporter.getInstance().report(scheduler.getScanTarget(), 154 | // startTime.getTime(), totalTasks.get() / scheduler.getScanPortNum(), 155 | // scheduler.getScanPortNum(), proxyIPNum.get(), consumedTime); 156 | 157 | log.info("Finished one scanning schedule, total tasks: {}, consumed time: {},file address:{}", 158 | totalTasks, consumedTime, outputTmpFile.getPath()); 159 | } 160 | } 161 | 162 | private class HTTPProxyRespHandler extends ChannelInboundHandlerAdapter { 163 | 164 | @Override 165 | public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { 166 | String responseText; 167 | if (msg instanceof HttpContent) { 168 | HttpContent content = (HttpContent) msg; 169 | ByteBuf buf = content.content(); 170 | responseText = buf.toString(io.netty.util.CharsetUtil.UTF_8); 171 | buf.release(); 172 | 173 | String proxyIP = (String) ctx.channel().attr(AttributeKey.valueOf("ip")).get(); 174 | ProxyType proxyType = checkProxyType(proxyIP, responseText.trim()); 175 | if (proxyType != null) { 176 | ProxyInfo proxyInfo; 177 | Integer port = (Integer) ctx.channel().attr(AttributeKey.valueOf("port")).get(); 178 | proxyInfo = new ProxyInfo(proxyIP, 179 | ProtocolType.HTTP.getType(), port, proxyType.getType(), System.currentTimeMillis()); 180 | log.info("FIND PROXY:" + proxyInfo.toString()); 181 | proxyIPNum.incrementAndGet(); 182 | //TODO 添加GEO数据库 183 | // proxyInfo.setLocation(IPLocationUtils.getLocation(proxyInfo.getIp())); 184 | proxyInfoDao.append(proxyInfo, outputTmpFile); 185 | } 186 | } 187 | 188 | finish(ctx); 189 | ctx.channel().close(); 190 | } 191 | 192 | @Override 193 | public void channelActive(ChannelHandlerContext ctx) throws Exception { 194 | // Send HTTP request 195 | URI uri = new URI(getProxyCheckerURL()); 196 | int port = uri.getPort(); 197 | String host = uri.getHost() + ":" + String.valueOf(port); 198 | 199 | HttpRequest request = new DefaultHttpRequest( 200 | HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath()); 201 | request.headers().set(HttpHeaderNames.HOST, host); 202 | request.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE); 203 | request.headers().set(HttpHeaderNames.PROXY_CONNECTION, HttpHeaderValues.KEEP_ALIVE); 204 | 205 | 206 | ctx.channel().writeAndFlush(request); 207 | } 208 | 209 | @Override 210 | public void channelInactive(ChannelHandlerContext ctx) throws Exception { 211 | finish(ctx); 212 | ctx.fireChannelInactive(); 213 | } 214 | 215 | @Override 216 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { 217 | finish(ctx); 218 | ctx.channel().close(); 219 | } 220 | 221 | @Override 222 | public void channelRegistered(ChannelHandlerContext ctx) throws Exception { 223 | taskCounter.incrementAndGet(); 224 | ctx.fireChannelRegistered(); 225 | } 226 | 227 | @Override 228 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { 229 | finish(ctx); 230 | ctx.fireChannelUnregistered(); 231 | } 232 | } 233 | 234 | private class HttpClientOutboundHandler extends ChannelOutboundHandlerAdapter { 235 | @Override 236 | public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { 237 | finish(ctx); 238 | ctx.close(promise); 239 | } 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/util/IPPoolUtils.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.util; 2 | 3 | import com.noisy.proxy.ProxyScanner; 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.*; 9 | import java.util.regex.Matcher; 10 | import java.util.regex.Pattern; 11 | 12 | /** 13 | * Created by kevin on 5/25/16. 14 | */ 15 | public class IPPoolUtils { 16 | private final static Logger log = LoggerFactory.getLogger(IPPoolUtils.class); 17 | private static final Pattern PATTERN = Pattern.compile( 18 | "^(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.){3}([01]?\\d\\d?|2[0-4]\\d|25[0-5])$"); 19 | private final static Pattern ipreg = Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"); 20 | 21 | public static Set retrieveIPFromText(String text) { 22 | if (text == null) { 23 | return null; 24 | } 25 | 26 | Matcher matcher = ipreg.matcher(text); 27 | Set ipSet = new HashSet<>(); 28 | while (matcher.find()) { 29 | ipSet.add(matcher.group()); 30 | } 31 | 32 | return ipSet; 33 | } 34 | 35 | public static boolean validateIP(final String ip) { 36 | return PATTERN.matcher(ip).matches(); 37 | } 38 | 39 | /** 40 | * Get IPSegments from a normal IP segment expression list, IP with mask bits, eg. 220.154.0.0/15 41 | * 42 | * @param ipWithMaskBitsList 43 | * @return 44 | */ 45 | public static List getIPSegments(Collection ipWithMaskBitsList) { 46 | if (ipWithMaskBitsList == null) { 47 | throw new IllegalArgumentException("The given IP segment expression list is null."); 48 | } 49 | Random random = new Random(System.currentTimeMillis()); 50 | 51 | List ipSegments = new ArrayList<>(); 52 | for (String ipWithMaskBits : ipWithMaskBitsList) { 53 | IPSegment ipSegment = getIPSegment(ipWithMaskBits); 54 | ipSegment.setRandomIndex(random.nextInt()); 55 | //ipSegment.setIpWithMaskBits(ipWithMaskBits); 56 | if (needAdds(ipSegment, ipSegments)) { 57 | ipSegments.add(ipSegment); 58 | } 59 | } 60 | Collections.sort(ipSegments); 61 | 62 | return ipSegments; 63 | } 64 | 65 | private static boolean needAdds(final IPSegment ipSegment, final List ipSegments) { 66 | Iterator iterator = ipSegments.iterator(); 67 | while (iterator.hasNext()) { 68 | IPSegment segment = iterator.next(); 69 | // if the given IP segment has been include 70 | if (ipSegment.getStartLongIP() >= segment.getStartLongIP() 71 | && ipSegment.getEndLongIP() <= segment.getEndLongIP()) { 72 | return false; 73 | } 74 | 75 | // if the given IP segment include an exists IP segment in the list, then remove the exists one. 76 | if (ipSegment.getStartLongIP() <= segment.getStartLongIP() 77 | && ipSegment.getEndLongIP() >= segment.getEndLongIP()) { 78 | iterator.remove(); 79 | } 80 | } 81 | 82 | return true; 83 | } 84 | 85 | /** 86 | * Get IPSegment from a normal IP segment expression, IP with mask bits, eg. 220.154.0.0/15 87 | * 88 | * @param ipWithMaskBits 89 | * @return 90 | */ 91 | public static IPSegment getIPSegment(String ipWithMaskBits) { 92 | if (StringUtils.isEmpty(ipWithMaskBits)) { 93 | log.error("The given parameter is empty or null."); 94 | return null; 95 | } 96 | 97 | String[] ipSegTmp = ipWithMaskBits.trim().split("\\/"); 98 | if (ipSegTmp.length != 2 || !validateIP(ipSegTmp[0])) { 99 | log.error("The given IP segments expression is invalid."); 100 | return null; 101 | } 102 | 103 | return getEndIP(ipSegTmp[0], Integer.valueOf(ipSegTmp[1])); 104 | } 105 | 106 | /** 107 | * According to the start IP address and the bits number of net mask to calculate the end IP address. 108 | * 109 | * @param startIP 110 | * @param netmask 111 | * @return 112 | */ 113 | public static IPSegment getEndIP(String startIP, int netmask) { 114 | return getEndIP(startIP, getMask(netmask)); 115 | } 116 | 117 | /** 118 | * Return the net mask with the given bits of net mask 119 | * 120 | * @param masks 121 | * @return 122 | */ 123 | public static String getMask(int masks) { 124 | if (masks == 1) 125 | return "128.0.0.0"; 126 | else if (masks == 2) 127 | return "192.0.0.0"; 128 | else if (masks == 3) 129 | return "224.0.0.0"; 130 | else if (masks == 4) 131 | return "240.0.0.0"; 132 | else if (masks == 5) 133 | return "248.0.0.0"; 134 | else if (masks == 6) 135 | return "252.0.0.0"; 136 | else if (masks == 7) 137 | return "254.0.0.0"; 138 | else if (masks == 8) 139 | return "255.0.0.0"; 140 | else if (masks == 9) 141 | return "255.128.0.0"; 142 | else if (masks == 10) 143 | return "255.192.0.0"; 144 | else if (masks == 11) 145 | return "255.224.0.0"; 146 | else if (masks == 12) 147 | return "255.240.0.0"; 148 | else if (masks == 13) 149 | return "255.248.0.0"; 150 | else if (masks == 14) 151 | return "255.252.0.0"; 152 | else if (masks == 15) 153 | return "255.254.0.0"; 154 | else if (masks == 16) 155 | return "255.255.0.0"; 156 | else if (masks == 17) 157 | return "255.255.128.0"; 158 | else if (masks == 18) 159 | return "255.255.192.0"; 160 | else if (masks == 19) 161 | return "255.255.224.0"; 162 | else if (masks == 20) 163 | return "255.255.240.0"; 164 | else if (masks == 21) 165 | return "255.255.248.0"; 166 | else if (masks == 22) 167 | return "255.255.252.0"; 168 | else if (masks == 23) 169 | return "255.255.254.0"; 170 | else if (masks == 24) 171 | return "255.255.255.0"; 172 | else if (masks == 25) 173 | return "255.255.255.128"; 174 | else if (masks == 26) 175 | return "255.255.255.192"; 176 | else if (masks == 27) 177 | return "255.255.255.224"; 178 | else if (masks == 28) 179 | return "255.255.255.240"; 180 | else if (masks == 29) 181 | return "255.255.255.248"; 182 | else if (masks == 30) 183 | return "255.255.255.252"; 184 | else if (masks == 31) 185 | return "255.255.255.254"; 186 | else if (masks == 32) 187 | return "255.255.255.255"; 188 | return ""; 189 | } 190 | 191 | /** 192 | * According to the start IP address and the net mask to calculate the end IP address. 193 | * 194 | * @param startIP 195 | * @param netmask 196 | * @return 197 | */ 198 | public static IPSegment getEndIP(String startIP, String netmask) { 199 | String[] start = Negation(startIP, netmask).split("\\."); 200 | String tmpStartIP = start[0] + "." + start[1] + "." + start[2] + "." + (Integer.valueOf(start[3]) + 1); 201 | String tmpEndIP = TaskOR(Negation(startIP, netmask), netmask); 202 | 203 | return new IPSegment(tmpStartIP, tmpEndIP, netmask); 204 | } 205 | 206 | /** 207 | * Return the negation of the given start IP address and net mask. 208 | * 209 | * @param startIP 210 | * @param netmask 211 | * @return 212 | */ 213 | private static String Negation(String startIP, String netmask) { 214 | String[] temp1 = startIP.trim().split("\\."); 215 | String[] temp2 = netmask.trim().split("\\."); 216 | int[] rets = new int[4]; 217 | for (int i = 0; i < 4; i++) { 218 | rets[i] = Integer.parseInt(temp1[i]) & Integer.parseInt(temp2[i]); 219 | } 220 | return rets[0] + "." + rets[1] + "." + rets[2] + "." + rets[3]; 221 | } 222 | 223 | private static String TaskOR(String StartIP, String netmask) { 224 | String[] temp1 = StartIP.trim().split("\\."); 225 | String[] temp2 = netmask.trim().split("\\."); 226 | int[] rets = new int[4]; 227 | for (int i = 0; i < 4; i++) { 228 | rets[i] = 255 - (Integer.parseInt(temp1[i]) ^ Integer.parseInt(temp2[i])); 229 | } 230 | return rets[0] + "." + rets[1] + "." + rets[2] + "." + (rets[3] - 1); 231 | } 232 | 233 | /** 234 | * Return address pool size by the bits of net mask. 235 | */ 236 | public static int getPoolMax(int netmask) { 237 | if (netmask <= 0 || netmask >= 32) { 238 | return 0; 239 | } 240 | int bits = 32 - netmask; 241 | return (int) Math.pow(2, bits) - 2; 242 | } 243 | 244 | /** 245 | * Convert the net mask to bits of net mask. 246 | */ 247 | public static int getNetMask(String netmarks) { 248 | StringBuffer sbf; 249 | String str; 250 | int inetmask = 0, count = 0; 251 | String[] ipList = netmarks.split("\\."); 252 | for (int n = 0; n < ipList.length; n++) { 253 | sbf = toBin(Integer.parseInt(ipList[n])); 254 | str = sbf.reverse().toString(); 255 | count = 0; 256 | for (int i = 0; i < str.length(); i++) { 257 | i = str.indexOf('1', i); 258 | if (i == -1) { 259 | break; 260 | } 261 | count++; 262 | } 263 | inetmask += count; 264 | } 265 | return inetmask; 266 | } 267 | 268 | private static StringBuffer toBin(int x) { 269 | StringBuffer result = new StringBuffer(); 270 | result.append(x % 2); 271 | x /= 2; 272 | while (x > 0) { 273 | result.append(x % 2); 274 | x /= 2; 275 | } 276 | return result; 277 | } 278 | 279 | 280 | public static void main(String[] args) { 281 | 282 | String filePath = "/Users/xulei2/data/ip_data/InlandIPSegments.txt"; 283 | Map iPSegments = ProxyScanner.readIPSegments(filePath); 284 | List ipSegmentList = IPPoolUtils.getIPSegments(iPSegments.values()); 285 | int count = 0; 286 | while (!ipSegmentList.isEmpty()){ 287 | Iterator iterator = ipSegmentList.iterator(); 288 | while (iterator.hasNext()) { 289 | IPSegment ipSegment = iterator.next(); 290 | if (ipSegment.hasNextIP()) { 291 | String proxyIP = ipSegment.getNextStringIP(); 292 | if (!IPFilterUtils.getInstance().needFilter(proxyIP)) { 293 | log.info("proxy detect start...proxyIP:{} ### {}",proxyIP,count); 294 | count+=1; 295 | } 296 | } else { 297 | iterator.remove(); 298 | } 299 | } 300 | } 301 | 302 | log.info("total proxy count: {}",count); 303 | } 304 | } 305 | 306 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/detector/Socks4ProxyDetector.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.detector; 2 | 3 | import com.noisy.proxy.TaskScheduler; 4 | import com.noisy.proxy.dao.ProxyInfoDao; 5 | import com.noisy.proxy.dao.ProxyInfoDaoFilempl; 6 | import com.noisy.proxy.entity.ProtocolType; 7 | import com.noisy.proxy.entity.ProxyInfo; 8 | import com.noisy.proxy.entity.ProxyType; 9 | import io.netty.bootstrap.Bootstrap; 10 | import io.netty.buffer.ByteBuf; 11 | import io.netty.channel.*; 12 | import io.netty.channel.nio.NioEventLoopGroup; 13 | import io.netty.channel.socket.SocketChannel; 14 | import io.netty.channel.socket.nio.NioSocketChannel; 15 | import io.netty.handler.codec.http.*; 16 | import io.netty.handler.proxy.Socks4ProxyHandler; 17 | import io.netty.handler.timeout.ReadTimeoutHandler; 18 | import io.netty.handler.timeout.WriteTimeoutHandler; 19 | import io.netty.util.AttributeKey; 20 | import lombok.extern.slf4j.Slf4j; 21 | import org.apache.commons.io.FileUtils; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | import java.net.InetSocketAddress; 26 | import java.net.URI; 27 | import java.net.URISyntaxException; 28 | import java.net.UnknownHostException; 29 | import java.nio.charset.Charset; 30 | import java.util.Calendar; 31 | import java.util.concurrent.Semaphore; 32 | import java.util.concurrent.TimeUnit; 33 | import java.util.concurrent.atomic.AtomicLong; 34 | 35 | /** 36 | * @author lei.X 37 | * @date 2019/5/29 38 | */ 39 | @Slf4j 40 | public class Socks4ProxyDetector extends AbstractProxyDetector { 41 | 42 | private final static int THREAD_POOL_SIZE_CORES_MULTIPLE = 2; 43 | private final static int nThreads = THREAD_POOL_SIZE_CORES_MULTIPLE * Runtime.getRuntime().availableProcessors(); 44 | 45 | private final ProxyInfoDao proxyInfoDao = new ProxyInfoDaoFilempl(); 46 | private EventLoopGroup workerGroup; 47 | private Calendar startTime = Calendar.getInstance(); 48 | private final AtomicLong taskCounter = new AtomicLong(0); 49 | private final AtomicLong totalTasks = new AtomicLong(0); 50 | private final AtomicLong proxyIPNum = new AtomicLong(0); 51 | // The TCP connection limits of the system, must small than 500000 52 | private final int tcpConnLimits = 2000; 53 | private final Semaphore semaphore = new Semaphore(tcpConnLimits); 54 | private final Semaphore canStartNextSchSem = new Semaphore(1); 55 | private Bootstrap bootstrap; 56 | private String outputTmpFilePath; 57 | private File outputTmpFile; 58 | private TaskScheduler scheduler; 59 | private volatile boolean isReported; 60 | 61 | 62 | public Socks4ProxyDetector(TaskScheduler scheduler) { 63 | this.scheduler = scheduler; 64 | 65 | workerGroup = new NioEventLoopGroup(nThreads); 66 | 67 | 68 | bootstrap = new Bootstrap(); 69 | bootstrap.group(workerGroup); 70 | bootstrap.channel(NioSocketChannel.class); 71 | bootstrap.option(ChannelOption.SO_REUSEADDR, true); 72 | bootstrap.option(ChannelOption.TCP_NODELAY, true); 73 | bootstrap.option(ChannelOption.SO_KEEPALIVE, false); 74 | bootstrap.option(ChannelOption.SO_SNDBUF, 1024); 75 | bootstrap.option(ChannelOption.SO_RCVBUF, 4096); 76 | bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator()); 77 | bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getTimeout()); 78 | 79 | outputTmpFilePath = getOutputTmpFilePath(scheduler.getName()); 80 | outputTmpFile = new File(outputTmpFilePath); 81 | if (!outputTmpFile.getParentFile().exists()) { 82 | outputTmpFile.getParentFile().mkdirs(); 83 | } 84 | 85 | isReported = false; 86 | } 87 | 88 | @Override 89 | public void detect(String ip, int port) { 90 | try { 91 | semaphore.acquire(); 92 | } catch (InterruptedException e) { 93 | log.warn("An exception occurred when creating a detection task, ex: {}", e); 94 | } 95 | 96 | 97 | bootstrap.handler(new ChannelInitializer() { 98 | @Override 99 | public void initChannel(SocketChannel ch) throws Exception { 100 | ChannelPipeline p = ch.pipeline(); 101 | 102 | p.addLast(new HttpClientCodec()); 103 | p.addLast(new HttpContentDecompressor()); 104 | p.addLast(new HttpObjectAggregator(10_485_760)); 105 | p.addLast(new WriteTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 106 | p.addLast(new ReadTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 107 | p.addLast(new HttpClientOutboundHandler()); 108 | p.addLast(new SocksProxyRespHandler()); 109 | p.addFirst(new Socks4ProxyHandler(new InetSocketAddress(ip, port))); 110 | } 111 | }); 112 | 113 | 114 | // Connecting the proxy 115 | try { 116 | ChannelFuture f = bootstrap.connect(ip, port); 117 | f.channel().attr(AttributeKey.valueOf("ip")).set(ip); 118 | f.channel().attr(AttributeKey.valueOf("port")).set(port); 119 | 120 | long leftTasks = taskCounter.get(); 121 | long tmpTotalTasks = totalTasks.incrementAndGet(); 122 | // if (((tmpTotalTasks == scheduler.getTotalTasks()) && (leftTasks % 100) == 0) 123 | // || ((tmpTotalTasks % 1000000) == 0)) { 124 | // log.info("Created tasks: {}, Left tasks: {}", tmpTotalTasks, leftTasks); 125 | // } 126 | } catch (Exception e) { 127 | log.error("An exception occurred when connecting to host {}:{}, ex: {}", ip, port, e); 128 | } 129 | } 130 | 131 | @Override 132 | public void reset() { 133 | try { 134 | canStartNextSchSem.acquire(); 135 | } catch (InterruptedException e) { 136 | log.error("An exception occurred when resetting the Socks4 proxy detector, ex: {}", e); 137 | } 138 | startTime.setTimeInMillis(System.currentTimeMillis()); 139 | totalTasks.set(taskCounter.get()); 140 | proxyIPNum.set(0); 141 | outputTmpFilePath = getOutputTmpFilePath(scheduler.getName()); 142 | outputTmpFile = new File(outputTmpFilePath); 143 | if (!outputTmpFile.getParentFile().exists()) { 144 | outputTmpFile.getParentFile().mkdirs(); 145 | } 146 | } 147 | 148 | 149 | private synchronized void finish(ChannelHandlerContext ctx) { 150 | if (ctx.channel().hasAttr(AttributeKey.valueOf("finished"))) { 151 | return; 152 | } 153 | 154 | semaphore.release(); 155 | ctx.channel().attr(AttributeKey.valueOf("finished")).set(true); 156 | 157 | long leftTasks = taskCounter.decrementAndGet(); 158 | if (leftTasks < 1) { 159 | log.info("Total tasks: {}, Left tasks: {}", totalTasks, taskCounter); 160 | canStartNextSchSem.release(); 161 | 162 | long consumedTime = System.currentTimeMillis() - startTime.getTimeInMillis(); 163 | // EmailReporter.getInstance().report(scheduler.getScanTarget(), 164 | // startTime.getTime(), totalTasks.get() / scheduler.getScanPortNum(), 165 | // scheduler.getScanPortNum(), proxyIPNum.get(), consumedTime); 166 | String distFilePath = retrieveOutputFilePath(outputTmpFilePath); 167 | try { 168 | if (outputTmpFile.exists()) 169 | FileUtils.moveFile(outputTmpFile, new File(distFilePath)); 170 | } catch (IOException e) { 171 | log.error("Failed to move the temporary file {} to the dist file {}. Exception: {}", 172 | outputTmpFile.getPath(), distFilePath, e); 173 | } 174 | log.info("Finished one scanning schedule, total tasks: {}, consumed time: {}", 175 | totalTasks, consumedTime); 176 | } 177 | } 178 | 179 | private class SocksProxyRespHandler extends ChannelInboundHandlerAdapter { 180 | 181 | @Override 182 | public void channelRead(ChannelHandlerContext ctx, Object msg) throws UnknownHostException { 183 | String responseText; 184 | if (msg instanceof FullHttpResponse) { 185 | FullHttpResponse httpResp = (FullHttpResponse) msg; 186 | 187 | 188 | ByteBuf content = httpResp.content(); 189 | responseText = content.toString(Charset.forName("UTF-8")); 190 | } else { 191 | responseText = msg.toString(); 192 | } 193 | String proxyIP = (String) ctx.channel().attr(AttributeKey.valueOf("ip")).get(); 194 | 195 | ProxyType proxyType = checkProxyType(proxyIP, responseText.trim()); 196 | if (proxyType != null) { 197 | ProxyInfo proxyInfo; 198 | Integer port = (Integer) ctx.channel().attr(AttributeKey.valueOf("port")).get(); 199 | proxyInfo = new ProxyInfo(proxyIP, 200 | ProtocolType.SOCKS_V4.getType(), port, proxyType.getType(), System.currentTimeMillis()); 201 | proxyIPNum.incrementAndGet(); 202 | log.info("FIND PROXY:" + proxyInfo.toString()); 203 | // proxyInfo.setLocation(IPLocationUtils.getLocation(proxyInfo.getIp())); 204 | proxyInfoDao.append(proxyInfo, outputTmpFile); 205 | } 206 | 207 | finish(ctx); 208 | ctx.channel().close(); 209 | } 210 | 211 | @Override 212 | public void channelActive(ChannelHandlerContext ctx) { 213 | 214 | // HttpRequest request = new DefaultFullHttpRequest( 215 | // HttpVersion.HTTP_1_0, HttpMethod.GET, getProxyCheckerPath()); 216 | // request.headers().set("Host", getProxyCheckerHost() + ":" + getProxyCheckerPort()); 217 | // ctx.writeAndFlush(request); 218 | 219 | 220 | // Send HTTP request 221 | URI uri = null; 222 | try { 223 | uri = new URI(getProxyCheckerURL()); 224 | } catch (URISyntaxException e) { 225 | log.error(e.getMessage()); 226 | } 227 | int port = uri.getPort(); 228 | String host = uri.getHost() + ":" + String.valueOf(port); 229 | 230 | HttpRequest request = new DefaultHttpRequest( 231 | HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath()); 232 | request.headers().set(HttpHeaderNames.HOST, host); 233 | request.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE); 234 | request.headers().set(HttpHeaderNames.PROXY_CONNECTION, HttpHeaderValues.KEEP_ALIVE); 235 | 236 | ctx.channel().writeAndFlush(request); 237 | 238 | } 239 | 240 | @Override 241 | public void channelInactive(ChannelHandlerContext ctx) throws Exception { 242 | finish(ctx); 243 | ctx.fireChannelInactive(); 244 | } 245 | 246 | @Override 247 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 248 | ctx.channel().close(); 249 | finish(ctx); 250 | } 251 | 252 | @Override 253 | public void channelRegistered(ChannelHandlerContext ctx) throws Exception { 254 | taskCounter.incrementAndGet(); 255 | ctx.fireChannelRegistered(); 256 | } 257 | 258 | @Override 259 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { 260 | finish(ctx); 261 | ctx.fireChannelUnregistered(); 262 | } 263 | } 264 | 265 | private class HttpClientOutboundHandler extends ChannelOutboundHandlerAdapter { 266 | @Override 267 | public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { 268 | finish(ctx); 269 | ctx.close(promise); 270 | } 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/ProxyScanner.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy; 2 | 3 | import com.noisy.proxy.entity.InputType; 4 | import com.noisy.proxy.entity.ProtocolType; 5 | import org.apache.commons.io.FileUtils; 6 | import org.apache.commons.io.IOUtils; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.beans.factory.annotation.Value; 11 | import org.springframework.core.env.Environment; 12 | import org.springframework.stereotype.Component; 13 | 14 | import java.io.*; 15 | import java.text.DateFormat; 16 | import java.text.ParseException; 17 | import java.text.SimpleDateFormat; 18 | import java.util.*; 19 | import java.util.concurrent.Executors; 20 | import java.util.concurrent.ScheduledExecutorService; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | /** 24 | * @author lei.X 25 | * @date 2019/5/5 26 | */ 27 | 28 | @Component 29 | public class ProxyScanner { 30 | 31 | 32 | private final static Logger log = LoggerFactory.getLogger(ProxyScanner.class); 33 | 34 | private static Map inlandIPSegments; 35 | private static Map abroadIPSegments; 36 | private static Set ipList; 37 | private static Set ports; 38 | 39 | @Autowired 40 | private Environment env; 41 | @Value("${data.address}") 42 | private String dataAddress; 43 | @Value("${input_type}") 44 | // 设定初始类型(选择IP_LIST 还是 IP_SEGMENT) 45 | private String inputType; 46 | 47 | 48 | public void start() { 49 | 50 | 51 | String ipListFilePath = dataAddress + "IpList.txt"; 52 | String inlandFilePath = dataAddress + "InlandIPSegments.txt"; 53 | String abroadIpFilePath = dataAddress + "AbroadIPSegments.txt"; 54 | String portFilePath = env.getProperty("detection.ports.file"); 55 | 56 | 57 | // 扫描控制 58 | long inlandHTTPInterval = Integer.valueOf(env.getProperty("inland.http.scanner.interval")) * 60 * 60 * 1000; 59 | long inlandSocks4Interval = Integer.valueOf(env.getProperty("inland.socks4.scanner.interval")) * 60 * 60 * 1000; 60 | long inlandSocks5Interval = Integer.valueOf(env.getProperty("inland.socks5.scanner.interval")) * 60 * 60 * 1000; 61 | long abroadHTTPInterval = Integer.valueOf(env.getProperty("abroad.http.scanner.interval")) * 60 * 60 * 1000; 62 | long abroadSocks4Interval = Integer.valueOf(env.getProperty("abroad.socks4.scanner.interval")) * 60 * 60 * 1000; 63 | long abroadSocks5Interval = Integer.valueOf(env.getProperty("abroad.socks5.scanner.interval")) * 60 * 60 * 1000; 64 | 65 | 66 | log.info("Starting the scanner."); 67 | ScheduledExecutorService executor = Executors.newScheduledThreadPool(4); 68 | 69 | //读取IP list文件 70 | if (InputType.IP_LIST.isThisType(inputType)) { 71 | ipList = getIpListSet(ipListFilePath); 72 | } else { 73 | inlandIPSegments = readIPSegments(inlandFilePath); 74 | abroadIPSegments = readIPSegments(abroadIpFilePath); 75 | } 76 | // 读取port文件 77 | ports = getPortSet(portFilePath); 78 | 79 | if (inlandHTTPInterval >= 0) { 80 | scheduleInlandHTTPTask(executor, inlandHTTPInterval); 81 | } 82 | 83 | if (inlandSocks4Interval >= 0) { 84 | scheduleInlandSocks4Task(executor,inlandSocks4Interval); 85 | } 86 | 87 | if (inlandSocks5Interval >= 0) { 88 | scheduleInlandSocks5Task(executor,inlandSocks5Interval); 89 | } 90 | 91 | if (InputType.IP_SEGMENTS.isThisType(inputType)) { 92 | if (abroadHTTPInterval >= 0) { 93 | scheduleAbroadHTTPTask(executor,abroadHTTPInterval); 94 | } 95 | 96 | if (abroadSocks4Interval >= 0) { 97 | scheduleAbroadSocks4Task(executor,abroadSocks4Interval); 98 | } 99 | 100 | if (abroadSocks5Interval >= 0) { 101 | scheduleAbroadSocks5Task(executor,abroadSocks5Interval); 102 | } 103 | } 104 | 105 | log.info("Scanner started."); 106 | 107 | } 108 | 109 | 110 | private void scheduleInlandHTTPTask(ScheduledExecutorService executor, long inlandHTTPInterval) { 111 | log.info("Scheduling the inland HTTP proxy scheduler."); 112 | TaskScheduler scheduler = null; 113 | 114 | if (InputType.IP_LIST.isThisType(inputType)) { 115 | scheduler = new TaskScheduler("InlandHTTPProxy", "国内HTTP代理", ProtocolType.HTTP, ipList,ports); 116 | } else { 117 | scheduler = new TaskScheduler("InlandHTTPProxy", "国内HTTP代理", ProtocolType.HTTP, inlandIPSegments,ports); 118 | } 119 | //定义启动时间和启动间隔 120 | executor.scheduleAtFixedRate(scheduler, getInitDelay(inlandHTTPInterval, 121 | Integer.valueOf(env.getProperty("inland.http.scanner.start"))), inlandHTTPInterval, TimeUnit.MILLISECONDS); 122 | } 123 | 124 | 125 | private void scheduleInlandSocks4Task(ScheduledExecutorService executor,long inlandSocks4Interval){ 126 | TaskScheduler scheduler = null; 127 | if (InputType.IP_LIST.isThisType(inputType)) { 128 | scheduler = new TaskScheduler("InlandSocks4Proxy", "国内Socks4代理", 129 | ProtocolType.SOCKS_V4, ipList,ports); 130 | } else { 131 | scheduler = new TaskScheduler("InlandSocks4Proxy", "国内Socks4代理", 132 | ProtocolType.SOCKS_V4, inlandIPSegments,ports); 133 | } 134 | executor.scheduleAtFixedRate(scheduler, getInitDelay(inlandSocks4Interval, 135 | Integer.valueOf(env.getProperty("inland.socks4.scanner.start"))), inlandSocks4Interval, TimeUnit.MILLISECONDS); 136 | } 137 | 138 | private void scheduleInlandSocks5Task(ScheduledExecutorService executor,long inlandSocks5Interval){ 139 | TaskScheduler scheduler = null; 140 | if (InputType.IP_LIST.isThisType(inputType)) { 141 | scheduler = new TaskScheduler("InlandSocks5Proxy", "国内Socks5代理", 142 | ProtocolType.SOCKS_V5, ipList,ports); 143 | } else { 144 | scheduler = new TaskScheduler("InlandSocks5Proxy", "国内Socks5代理", 145 | ProtocolType.SOCKS_V5, inlandIPSegments,ports); 146 | } 147 | executor.scheduleAtFixedRate(scheduler, getInitDelay(inlandSocks5Interval, 148 | Integer.valueOf(env.getProperty("inland.socks5.scanner.start"))), inlandSocks5Interval, TimeUnit.MILLISECONDS); 149 | } 150 | 151 | private void scheduleAbroadHTTPTask(ScheduledExecutorService executor,long abroadHTTPInterval) { 152 | TaskScheduler scheduler = new TaskScheduler("AbroadHTTPProxy", "国外HTTP代理", 153 | ProtocolType.HTTP, abroadIPSegments,ports); 154 | executor.scheduleAtFixedRate(scheduler, getInitDelay(abroadHTTPInterval, 155 | Integer.valueOf(env.getProperty("abroad.http.scanner.start"))), abroadHTTPInterval, TimeUnit.MILLISECONDS); 156 | } 157 | 158 | private void scheduleAbroadSocks4Task(ScheduledExecutorService executor,long abroadSocks4Interval) { 159 | TaskScheduler scheduler = new TaskScheduler("abroadSocks4Proxy", "国外socks4代理", 160 | ProtocolType.SOCKS_V4, abroadIPSegments,ports); 161 | executor.scheduleAtFixedRate(scheduler, getInitDelay(abroadSocks4Interval, 162 | Integer.valueOf(env.getProperty("abroad.socks4.scanner.start"))), abroadSocks4Interval, TimeUnit.MILLISECONDS); 163 | } 164 | 165 | private void scheduleAbroadSocks5Task(ScheduledExecutorService executor,long abroadSocks5Interval) { 166 | TaskScheduler scheduler = new TaskScheduler("abroadSocks5Proxy", "国外socks5代理", 167 | ProtocolType.SOCKS_V5, abroadIPSegments,ports); 168 | executor.scheduleAtFixedRate(scheduler, getInitDelay(abroadSocks5Interval, 169 | Integer.valueOf(env.getProperty("abroad.socks5.scanner.start"))), abroadSocks5Interval, TimeUnit.MILLISECONDS); 170 | } 171 | 172 | 173 | 174 | 175 | public static Map readIPSegments(String ipSegFilePath) { 176 | log.info("Loading IP segments from file: {}", ipSegFilePath); 177 | Map ipSegments = new HashMap<>(); 178 | File file = FileUtils.getFile(ipSegFilePath); 179 | if (file == null) { 180 | log.warn("Cannot load the IP segments from file: {}", ipSegFilePath); 181 | return null; 182 | } 183 | 184 | try { 185 | BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), "utf-8")); 186 | List ipSegList = IOUtils.readLines(br); 187 | if (ipSegList != null && !ipSegList.isEmpty()) { 188 | int idx = 1; 189 | for (String ipSeg : ipSegList) { 190 | ipSegments.put(idx, ipSeg); 191 | idx += 1; 192 | } 193 | br.close(); 194 | } else { 195 | log.warn("The IP segments file is empty, file path: {}", ipSegFilePath); 196 | br.close(); 197 | return null; 198 | } 199 | } catch (IOException e) { 200 | log.warn("An exception occurred when loading IP segments from file: {}", ipSegFilePath); 201 | return null; 202 | } 203 | 204 | return ipSegments; 205 | } 206 | 207 | 208 | private static long getInitDelay(long interval, int startHour) { 209 | if (startHour < 0) { 210 | return 0; 211 | } 212 | 213 | if (startHour > 24) { 214 | startHour = startHour % 24; 215 | } 216 | 217 | long initDelay = getTimeMillis(startHour + ":00:00") - System.currentTimeMillis(); 218 | initDelay = initDelay > 0 ? initDelay : interval + initDelay; 219 | 220 | return initDelay; 221 | } 222 | 223 | 224 | private static long getTimeMillis(String time) { 225 | try { 226 | DateFormat dateFormat = new SimpleDateFormat("yy-MM-dd HH:mm:ss"); 227 | DateFormat dayFormat = new SimpleDateFormat("yy-MM-dd"); 228 | Date curDate = dateFormat.parse(dayFormat.format(new Date()) + " " + time); 229 | return curDate.getTime(); 230 | } catch (ParseException e) { 231 | e.printStackTrace(); 232 | } 233 | return 0; 234 | } 235 | 236 | // 读取IP list 237 | private Set getIpListSet(String ipListFilePath) { 238 | 239 | Set filterIpList; 240 | 241 | File proxyIPFile = new File(ipListFilePath); 242 | try { 243 | List proxyIPs = FileUtils.readLines(proxyIPFile, "utf-8"); 244 | if (proxyIPs != null) { 245 | filterIpList = new HashSet<>(); 246 | filterIpList.addAll(proxyIPs); 247 | return filterIpList; 248 | 249 | } 250 | } catch (IOException e) { 251 | log.error("Failed to load the IP list file: {}", ipListFilePath); 252 | return null; 253 | } 254 | 255 | if (ipList == null || ipList.isEmpty()) { 256 | log.error("The IP list file is empty: {}", ipListFilePath); 257 | return null; 258 | } 259 | 260 | return null; 261 | } 262 | 263 | private Set getPortSet(String portListFilePath) { 264 | Set filterPortList = new HashSet<>(); 265 | 266 | File portListFile = new File(portListFilePath); 267 | try { 268 | List portList = FileUtils.readLines(portListFile, "utf-8"); 269 | if (portList != null && !portList.isEmpty()) { 270 | for (String port : portList) { 271 | filterPortList.add(Integer.valueOf(port)); 272 | 273 | } 274 | return filterPortList; 275 | } 276 | } catch (IOException e) { 277 | log.error("Failed to load the port list file: {}", portListFilePath); 278 | return null; 279 | } 280 | 281 | if (ipList == null || ipList.isEmpty()) { 282 | log.error("The port list file is empty: {}", portListFilePath); 283 | return null; 284 | } 285 | 286 | return null; 287 | } 288 | 289 | } 290 | -------------------------------------------------------------------------------- /proxy-kidd/src/main/java/com/noisy/proxy/detector/Socks5ProxyDetector.java: -------------------------------------------------------------------------------- 1 | package com.noisy.proxy.detector; 2 | 3 | import com.noisy.proxy.TaskScheduler; 4 | import com.noisy.proxy.dao.ProxyInfoDao; 5 | import com.noisy.proxy.dao.ProxyInfoDaoFilempl; 6 | import com.noisy.proxy.entity.ProtocolType; 7 | import com.noisy.proxy.entity.ProxyInfo; 8 | import com.noisy.proxy.entity.ProxyType; 9 | import io.netty.bootstrap.Bootstrap; 10 | import io.netty.buffer.ByteBuf; 11 | import io.netty.channel.*; 12 | import io.netty.channel.epoll.Epoll; 13 | import io.netty.channel.epoll.EpollChannelOption; 14 | import io.netty.channel.nio.NioEventLoopGroup; 15 | import io.netty.channel.socket.SocketChannel; 16 | import io.netty.channel.socket.nio.NioSocketChannel; 17 | import io.netty.handler.codec.http.*; 18 | import io.netty.handler.proxy.Socks5ProxyHandler; 19 | import io.netty.handler.timeout.ReadTimeoutHandler; 20 | import io.netty.handler.timeout.WriteTimeoutHandler; 21 | import io.netty.util.AttributeKey; 22 | import lombok.extern.slf4j.Slf4j; 23 | import org.apache.commons.io.FileUtils; 24 | 25 | import java.io.File; 26 | import java.io.IOException; 27 | import java.net.InetSocketAddress; 28 | import java.net.URI; 29 | import java.net.URISyntaxException; 30 | import java.net.UnknownHostException; 31 | import java.nio.charset.Charset; 32 | import java.util.Calendar; 33 | import java.util.concurrent.Semaphore; 34 | import java.util.concurrent.TimeUnit; 35 | import java.util.concurrent.atomic.AtomicLong; 36 | 37 | /** 38 | * Created by lei.x on 5/24/19. 39 | */ 40 | 41 | @Slf4j 42 | public class Socks5ProxyDetector extends AbstractProxyDetector { 43 | 44 | 45 | private final static int THREAD_POOL_SIZE_CORES_MULTIPLE = 2; 46 | private final static int nThreads = THREAD_POOL_SIZE_CORES_MULTIPLE * Runtime.getRuntime().availableProcessors(); 47 | 48 | private final ProxyInfoDao proxyInfoDao = new ProxyInfoDaoFilempl(); 49 | private EventLoopGroup workerGroup; 50 | private Calendar startTime = Calendar.getInstance(); 51 | private final AtomicLong taskCounter = new AtomicLong(0); 52 | private final AtomicLong totalTasks = new AtomicLong(0); 53 | private final AtomicLong proxyIPNum = new AtomicLong(0); 54 | private final int tcpConnLimits = 2000; 55 | private final Semaphore semaphore = new Semaphore(tcpConnLimits); 56 | private final Semaphore canStartNextSchSem = new Semaphore(1); 57 | private Bootstrap bootstrap; 58 | private String outputTmpFilePath; 59 | private File outputTmpFile; 60 | private TaskScheduler scheduler; 61 | private volatile boolean isReported; 62 | 63 | public Socks5ProxyDetector(TaskScheduler scheduler) { 64 | this.scheduler = scheduler; 65 | 66 | workerGroup = new NioEventLoopGroup(nThreads); 67 | 68 | bootstrap = new Bootstrap(); 69 | bootstrap.group(workerGroup); 70 | bootstrap.channel(NioSocketChannel.class); 71 | bootstrap.option(ChannelOption.SO_REUSEADDR, true); 72 | bootstrap.option(ChannelOption.TCP_NODELAY, true); 73 | bootstrap.option(ChannelOption.SO_KEEPALIVE, false); 74 | bootstrap.option(ChannelOption.SO_SNDBUF, 1024); 75 | bootstrap.option(ChannelOption.SO_RCVBUF, 4096); 76 | bootstrap.option(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator()); 77 | bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getTimeout()); 78 | 79 | outputTmpFilePath = getOutputTmpFilePath(scheduler.getName()); 80 | outputTmpFile = new File(outputTmpFilePath); 81 | if (!outputTmpFile.getParentFile().exists()) { 82 | outputTmpFile.getParentFile().mkdirs(); 83 | } 84 | 85 | isReported = false; 86 | } 87 | 88 | public void reset() { 89 | try { 90 | canStartNextSchSem.acquire(); 91 | } catch (InterruptedException e) { 92 | log.error("An exception occurred when resetting the Socks4 proxy detector, ex: {}", e); 93 | } 94 | startTime.setTimeInMillis(System.currentTimeMillis()); 95 | totalTasks.set(taskCounter.get()); 96 | proxyIPNum.set(0); 97 | outputTmpFilePath = getOutputTmpFilePath(scheduler.getName()); 98 | outputTmpFile = new File(outputTmpFilePath); 99 | if (!outputTmpFile.getParentFile().exists()) { 100 | outputTmpFile.getParentFile().mkdirs(); 101 | } 102 | } 103 | 104 | @Override 105 | public void detect(final String ip, final int port) { 106 | try { 107 | semaphore.acquire(); 108 | } catch (InterruptedException e) { 109 | log.warn("An exception occurred when creating a detection task, ex: {}", e); 110 | } 111 | 112 | if (Epoll.isAvailable()) { 113 | bootstrap.handler(new ChannelInitializer() { 114 | @Override 115 | public void initChannel(SocketChannel ch) throws Exception { 116 | ch.config().setOption(EpollChannelOption.SO_REUSEPORT, true); 117 | ChannelPipeline p = ch.pipeline(); 118 | 119 | p.addLast(new HttpClientCodec()); 120 | p.addLast(new HttpContentDecompressor()); 121 | p.addLast(new HttpObjectAggregator(10_485_760)); 122 | p.addLast(new WriteTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 123 | p.addLast(new ReadTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 124 | p.addLast(new HttpClientOutboundHandler()); 125 | p.addLast(new SocksProxyRespHandler()); 126 | p.addFirst(new Socks5ProxyHandler(new InetSocketAddress(ip, port))); 127 | } 128 | }); 129 | } else { 130 | bootstrap.handler(new ChannelInitializer() { 131 | @Override 132 | public void initChannel(SocketChannel ch) throws Exception { 133 | ChannelPipeline p = ch.pipeline(); 134 | 135 | p.addLast(new HttpClientCodec()); 136 | p.addLast(new HttpContentDecompressor()); 137 | p.addLast(new HttpObjectAggregator(10_485_760)); 138 | p.addLast(new WriteTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 139 | p.addLast(new ReadTimeoutHandler(getTimeout(), TimeUnit.MILLISECONDS)); 140 | p.addLast(new HttpClientOutboundHandler()); 141 | p.addLast(new SocksProxyRespHandler()); 142 | p.addFirst(new Socks5ProxyHandler(new InetSocketAddress(ip, port))); 143 | } 144 | }); 145 | } 146 | 147 | // Connecting the proxy 148 | try { 149 | ChannelFuture f = bootstrap.connect(ip, port); 150 | f.channel().attr(AttributeKey.valueOf("ip")).set(ip); 151 | f.channel().attr(AttributeKey.valueOf("port")).set(port); 152 | 153 | long leftTasks = taskCounter.get(); 154 | long tmpTotalTasks = totalTasks.incrementAndGet(); 155 | // if (((tmpTotalTasks == scheduler.getTotalTasks()) && (leftTasks % 100) == 0) 156 | // || ((tmpTotalTasks % 1000000) == 0)) { 157 | // log.info("Created tasks: {}, Left tasks: {}", tmpTotalTasks, leftTasks); 158 | // } 159 | } catch (Exception e) { 160 | log.error("An exception occurred when connecting to host {}:{}, ex: {}", ip, port, e); 161 | } 162 | } 163 | 164 | private synchronized void finish(ChannelHandlerContext ctx) { 165 | if (ctx.channel().hasAttr(AttributeKey.valueOf("finished"))) { 166 | return; 167 | } 168 | 169 | semaphore.release(); 170 | ctx.channel().attr(AttributeKey.valueOf("finished")).set(true); 171 | 172 | long leftTasks = taskCounter.decrementAndGet(); 173 | if (leftTasks < 1) { 174 | log.info("Total tasks: {}, Left tasks: {}", totalTasks, taskCounter); 175 | canStartNextSchSem.release(); 176 | 177 | long consumedTime = System.currentTimeMillis() - startTime.getTimeInMillis(); 178 | String distFilePath = retrieveOutputFilePath(outputTmpFilePath); 179 | try { 180 | if (outputTmpFile.exists()) 181 | FileUtils.moveFile(outputTmpFile, new File(distFilePath)); 182 | } catch (IOException e) { 183 | log.error("Failed to move the temporary file {} to the dist file {}. Exception: {}", 184 | outputTmpFile.getPath(), distFilePath, e); 185 | } 186 | log.info("Finished one scanning schedule, total tasks: {}, consumed time: {}", 187 | totalTasks, consumedTime); 188 | } 189 | } 190 | 191 | private class SocksProxyRespHandler extends ChannelInboundHandlerAdapter { 192 | 193 | @Override 194 | public void channelRead(ChannelHandlerContext ctx, Object msg) throws UnknownHostException { 195 | String responseText; 196 | if (msg instanceof FullHttpResponse) { 197 | FullHttpResponse httpResp = (FullHttpResponse) msg; 198 | 199 | 200 | ByteBuf content = httpResp.content(); 201 | responseText = content.toString(Charset.forName("UTF-8")); 202 | } else { 203 | responseText = msg.toString(); 204 | } 205 | String proxyIP = (String) ctx.channel().attr(AttributeKey.valueOf("ip")).get(); 206 | 207 | ProxyType proxyType = checkProxyType(proxyIP, responseText.trim()); 208 | if (proxyType != null) { 209 | ProxyInfo proxyInfo; 210 | Integer port = (Integer) ctx.channel().attr(AttributeKey.valueOf("port")).get(); 211 | proxyInfo = new ProxyInfo(proxyIP, 212 | ProtocolType.SOCKS_V5.getType(), port, proxyType.getType(), System.currentTimeMillis()); 213 | proxyIPNum.incrementAndGet(); 214 | log.info("FIND PROXY:" + proxyInfo.toString()); 215 | // proxyInfo.setLocation(IPLocationUtils.getLocation(proxyInfo.getIp())); 216 | proxyInfoDao.append(proxyInfo, outputTmpFile); 217 | } 218 | 219 | finish(ctx); 220 | ctx.channel().close(); 221 | } 222 | 223 | @Override 224 | public void channelActive(ChannelHandlerContext ctx) { 225 | // Send HTTP request 226 | URI uri = null; 227 | try { 228 | uri = new URI(getProxyCheckerURL()); 229 | } catch (URISyntaxException e) { 230 | log.error(e.getMessage()); 231 | } 232 | int port = uri.getPort(); 233 | String host = uri.getHost() + ":" + String.valueOf(port); 234 | 235 | HttpRequest request = new DefaultHttpRequest( 236 | HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath()); 237 | request.headers().set(HttpHeaderNames.HOST, host); 238 | request.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE); 239 | request.headers().set(HttpHeaderNames.PROXY_CONNECTION, HttpHeaderValues.KEEP_ALIVE); 240 | 241 | ctx.channel().writeAndFlush(request); 242 | } 243 | 244 | @Override 245 | public void channelInactive(ChannelHandlerContext ctx) throws Exception { 246 | finish(ctx); 247 | ctx.fireChannelInactive(); 248 | } 249 | 250 | @Override 251 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 252 | ctx.channel().close(); 253 | finish(ctx); 254 | } 255 | 256 | @Override 257 | public void channelRegistered(ChannelHandlerContext ctx) throws Exception { 258 | taskCounter.incrementAndGet(); 259 | ctx.fireChannelRegistered(); 260 | } 261 | 262 | @Override 263 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { 264 | finish(ctx); 265 | ctx.fireChannelUnregistered(); 266 | } 267 | } 268 | 269 | private class HttpClientOutboundHandler extends ChannelOutboundHandlerAdapter { 270 | @Override 271 | public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { 272 | finish(ctx); 273 | ctx.close(promise); 274 | } 275 | } 276 | } 277 | --------------------------------------------------------------------------------