├── .gitignore
├── pom.xml
├── readme.md
└── src
    ├── main
        ├── java
        │   └── com
        │   │   └── caspar
        │   │       ├── ProxyServer.java
        │   │       ├── ProxyTask.java
        │   │       ├── bean
        │   │           └── HttpHeader.java
        │   │       ├── exception
        │   │           └── HttpHeaderConstructException.java
        │   │       └── util
        │   │           ├── HttpClientUtil.java
        │   │           ├── HttpHeaderBuilder.java
        │   │           └── ProxyUtil.java
        └── resources
        │   └── logback.xml
    └── test
        └── java
            └── com
                └── caspar
                    ├── someThing
                        ├── TestChar.java
                        └── TestSocket.java
                    └── util
                        └── TestHttpHeaderBuilder.java


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Created by .ignore support plugin (hsz.mobi)
 2 | ### Java template
 3 | # Compiled class file
 4 | *.class
 5 | 
 6 | # Log file
 7 | *.log
 8 | 
 9 | .idea
10 | 
11 | # BlueJ files
12 | *.ctxt
13 | 
14 | # Mobile Tools for Java (J2ME)
15 | .mtj.tmp/
16 | 
17 | # Package Files #
18 | *.jar
19 | *.war
20 | *.nar
21 | *.ear
22 | *.zip
23 | *.tar.gz
24 | *.rar
25 | 
26 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
27 | hs_err_pid*
28 | 
29 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>com.caspar</groupId>
 8 |     <artifactId>proxyServer</artifactId>
 9 |     <version>1.0</version>
10 | 
11 |     <build>
12 |         <plugins>
13 |             <plugin>
14 |                 <groupId>org.apache.maven.plugins</groupId>
15 |                 <artifactId>maven-compiler-plugin</artifactId>
16 |                 <configuration>
17 |                     <source>1.8</source>
18 |                     <target>1.8</target>
19 |                 </configuration>
20 |             </plugin>
21 |         </plugins>
22 |     </build>
23 | 
24 |     <dependencies>
25 |         <dependency>
26 |             <groupId>org.mockito</groupId>
27 |             <artifactId>mockito-core</artifactId>
28 |             <version>2.2.28</version>
29 |             <scope>test</scope>
30 |         </dependency>
31 | 
32 |         <dependency>
33 |             <groupId>org.apache.httpcomponents</groupId>
34 |             <artifactId>httpcore</artifactId>
35 |             <version>4.4.10</version>
36 |         </dependency>
37 | 
38 |         <dependency>
39 |             <groupId>org.apache.httpcomponents</groupId>
40 |             <artifactId>httpclient</artifactId>
41 |             <version>4.5.6</version>
42 |         </dependency>
43 | 
44 |         <dependency>
45 |             <groupId>com.alibaba</groupId>
46 |             <artifactId>fastjson</artifactId>
47 |             <version>1.2.68</version>
48 |         </dependency>
49 | 
50 |         <dependency>
51 |             <groupId>junit</groupId>
52 |             <artifactId>junit</artifactId>
53 |             <version>4.13.1</version>
54 |             <scope>test</scope>
55 |         </dependency>
56 | 
57 |         <dependency>
58 |             <groupId>ch.qos.logback</groupId>
59 |             <artifactId>logback-classic</artifactId>
60 |             <version>1.1.7</version>
61 |         </dependency>
62 | 
63 | 
64 | 
65 |     </dependencies>
66 | 
67 | </project>


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | # ip代理池
 2 | 本项目其实就是个简单的代理服务器，经过我小小的修改。加了个代理池进来。
 3 | 渗透、爬虫的时候很容易就会把自己ip给ban了所以就需要ip代理池了。
 4 | ![ippool][1]
 5 | ## 使用
 6 | 
 7 | 1、启动ip代理池
 8 | 先把这个项目跑起来
 9 | https://github.com/jhao104/proxy_pool  
10 | 
11 | 2、启动代理服务器
12 | 
13 | 默认监听8080  
14 | java -jar proxyServer.jar
15 | 
16 | 自定义监听端口  
17 | java -jar proxyServer.jar 9090
18 | 
19 | 3、设置代理    
20 | 设置好代理后你就发现每次请求的ip都不一样。
21 | 
22 | ## 配置
23 | 简单说一下proxy_pool需要做的一些配置。   
24 | 1、如果有更好的代理网站，或者是你买了代理ip。可以在根目录的/fetcher/proxyFetcher.py里面自己写个方法去爬，该方法需要以生成器(yield)形式返回host:ip，然后在根目录
25 | 的setting.py文件里面，把自己写方法名添加进去。   
26 | 
27 | 2、redis默认装好是没密码的，需要修改setting.py,改为DB_CONN='redis://@127.0.0.1:6379/0'一般全部默认即可    
28 | 
29 | 3、修改超时，把setting.py中的VERIFY_TIMEOUT改小点(3)。默认10秒，这种ip基本用不成。   
30 | 
31 | 4、已发布版的代理服务器，用的是proxy_pool默认端口。而且代理池需要和代理服务器跑在一起。如不在一台机器上可修改ProxyUtil类里面getProxy方法的url。  
32 | 
33 | ## 最后
34 | 博客：http://www.safe6.cn/      
35 | 公众号：safe6安全的成长日记            
36 | 整合好的我已经打包放公众号，需要的自取(回复：ip代理池)    
37 | ![safe6][2]
38 | 
39 | ## 感谢开源项目
40 | https://github.com/casparhuan/proxyServer             
41 | https://github.com/jhao104/proxy_pool
42 | 
43 | 
44 |   [1]: http://qiniu.safe6.cn/ipPool.jpg
45 |   [2]: http://qiniu.safe6.cn/qrcode.jpg
46 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/ProxyServer.java:
--------------------------------------------------------------------------------
 1 | package com.caspar;
 2 | 
 3 | import org.slf4j.Logger;
 4 | import org.slf4j.LoggerFactory;
 5 | 
 6 | import java.io.IOException;
 7 | import java.net.InetSocketAddress;
 8 | import java.net.Proxy;
 9 | import java.net.ServerSocket;
10 | import java.net.Socket;
11 | import java.util.concurrent.ExecutorService;
12 | import java.util.concurrent.Executors;
13 | 
14 | /**
15 |  * Created by casparhuan on 2016/12/3.
16 |  */
17 | public class ProxyServer {
18 | 
19 |     private static Logger logger = LoggerFactory.getLogger(ProxyServer.class.getName());
20 |     private static boolean RUN_FLAG = true;
21 | 
22 |     public static void main(String[] args) throws IOException {
23 |         logger.debug("starting proxyServer...");
24 |         int listenPort = 8080;
25 |         if(args.length!=0){
26 |             try {
27 |                 listenPort = Integer.parseInt(args[0]);
28 |             }catch (Exception e){
29 |                 e.printStackTrace();
30 |                 logger.error(e.getMessage());
31 |                 listenPort = 8080;
32 |                 logger.info("由于输入参数的端口错误，仍然监听端口"+listenPort);
33 |             }
34 |         }
35 |         ServerSocket serverSocket = new ServerSocket(listenPort);
36 |         final ExecutorService executor = Executors.newCachedThreadPool();
37 | 
38 |         logger.info("开始监听端口："+listenPort);
39 | 
40 |         while (RUN_FLAG) {
41 |             try {
42 |                 Socket socket = serverSocket.accept();
43 |                 socket.setKeepAlive(true);
44 |                 socket.setSoTimeout(10000);
45 |                 executor.submit(new ProxyTask(socket));
46 |             } catch (Exception e) {
47 |                 e.printStackTrace();
48 |                 logger.error(e.getMessage());
49 |             }
50 |         }
51 | 
52 |         logger.info("服务结束");
53 |     }
54 | 
55 | }
56 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/ProxyTask.java:
--------------------------------------------------------------------------------
  1 | package com.caspar;
  2 | 
  3 | import com.caspar.bean.HttpHeader;
  4 | import com.caspar.exception.HttpHeaderConstructException;
  5 | import com.caspar.util.HttpHeaderBuilder;
  6 | import com.caspar.util.ProxyUtil;
  7 | import org.slf4j.Logger;
  8 | import org.slf4j.LoggerFactory;
  9 | 
 10 | import java.io.IOException;
 11 | import java.io.InputStream;
 12 | import java.io.OutputStream;
 13 | import java.net.InetSocketAddress;
 14 | import java.net.Proxy;
 15 | import java.net.Socket;
 16 | import java.net.SocketTimeoutException;
 17 | import java.util.concurrent.*;
 18 | import java.util.concurrent.atomic.AtomicLong;
 19 | 
 20 | /**
 21 |  * Created by casparhuan on 2016/12/3.
 22 |  */
 23 | public class ProxyTask implements Callable<Void> {
 24 |     private Socket inSocket;//使用代理的socket
 25 |     private Socket targetSocket;//目标访问网站的socket
 26 |     private Logger logger = LoggerFactory.getLogger(ProxyServer.class.getName());
 27 |     private static final ExecutorService executor = Executors.newCachedThreadPool();
 28 |     private volatile AtomicLong targetFileSize = new AtomicLong(0);
 29 |     private volatile AtomicLong clintGetFileSize = new AtomicLong(0);
 30 | 
 31 | 
 32 |     private static final String AUTHORED = "HTTP/1.1 200 Connection established";
 33 | 
 34 |     public ProxyTask(Socket socket) {
 35 |         this.inSocket = socket;
 36 |     }
 37 | 
 38 |     /**
 39 |      * 代理
 40 |      *
 41 |      * @return
 42 |      * @throws Exception
 43 |      */
 44 |     @Override
 45 |     public Void call() {
 46 |         // System.getProperties().put("socksProxySet ", "true ");
 47 |         //System.getProperties().put("socksProxyHost ", "127.0.0.1");
 48 |         //System.getProperties().put("socksProxyPort ", 8882);
 49 | 
 50 |         logger.debug("ip：" + inSocket.getInetAddress() + "  ,port:" + inSocket.getPort() + "  使用代理");
 51 |         InputStream isClient = null;
 52 |         OutputStream osClient = null;
 53 |         HttpHeader httpHeader = null;
 54 |         InputStream isTarget = null;
 55 |         OutputStream osTarget = null;
 56 | 
 57 |         try {
 58 |             isClient = inSocket.getInputStream();
 59 |             osClient = inSocket.getOutputStream();
 60 |             httpHeader = new HttpHeaderBuilder(isClient).build();
 61 |             logger.info("get reporter ：" + httpHeader);
 62 | 
 63 |             //socket = new Socket(proxy);
 64 |             //socket.connect(new InetSocketAddress(ip, port));//服务器的ip及地址
 65 |             //连接目标网站
 66 |             //targetSocket = new Socket(httpHeader.getHost(), httpHeader.getPort());
 67 |             targetSocket = new Socket(ProxyUtil.getProxy());
 68 |             targetSocket.connect(new InetSocketAddress(httpHeader.getHost(), httpHeader.getPort()));
 69 |             targetSocket.setSoTimeout(5000);
 70 |             targetSocket.setKeepAlive(true);
 71 |             isTarget = targetSocket.getInputStream();
 72 |             osTarget = targetSocket.getOutputStream();
 73 | 
 74 |             //报文头部发送
 75 |             if (HttpHeader.METHOD_CONNECTION.equals(httpHeader.getMethod().trim().toUpperCase())) {
 76 |                 //https访问
 77 |                 osClient.write((AUTHORED + "\r\n\r\n").getBytes());
 78 |                 osClient.flush();
 79 |             }
 80 |             else if(HttpHeader.METHOD_GET.equals(httpHeader.getMethod().trim().toUpperCase())){
 81 |                 //发送报文的请求头还有header
 82 |                 osTarget.write(httpHeader.getHttpHedaer().getBytes());
 83 |                 osTarget.write("\r\n".getBytes());
 84 |                 osTarget.flush();
 85 |             }
 86 |             else {
 87 |                 osTarget.write(httpHeader.getHttpHedaer().getBytes());
 88 |                 osTarget.flush();
 89 |             }
 90 | 
 91 | 
 92 |             //从目标网站服务器读取数据，传输给使用代理服务器的客户端
 93 |             InputStream finalIsTarget = isTarget;
 94 |             OutputStream finalOsClient = osClient;
 95 |             Callable<Void> readFromTargetToClientCallable = new Callable<Void>() {
 96 |                 @Override
 97 |                 public Void call() throws IOException {
 98 |                     logger.debug("从服务器端读取数据到客户端");
 99 |                     readFromTargetToClient(finalIsTarget, finalOsClient);
100 |                     logger.info("结束");
101 |                     return null;
102 |                 }
103 |             };
104 |             Future<Void> future = executor.submit(readFromTargetToClientCallable);
105 | 
106 |             //接受参数或者发送post正文内容
107 |             logger.debug("发送数据到目标服务器");
108 |             transferDataToTarget(isClient, osTarget);
109 | 
110 |             try {
111 |                 future.get();
112 |             } catch (InterruptedException e) {
113 |                 e.printStackTrace();
114 |             } catch (ExecutionException e) {
115 |                 e.printStackTrace();
116 |             }
117 | 
118 |             logger.info("ending...");
119 |         } catch (HttpHeaderConstructException e) {
120 |             e.printStackTrace();
121 |             logger.error(e.getMessage());
122 |             //返回固定格式
123 |             if (osClient != null) {
124 |                 try {
125 |                     osClient.write(badRequest().toString().getBytes());
126 |                 } catch (IOException e1) {
127 |                     e1.printStackTrace();
128 |                     logger.error(e1.getMessage());
129 |                 }
130 |             }
131 |             return null;
132 |         } catch (IOException ioe) {
133 |             ioe.printStackTrace();
134 |             logger.error(ioe.getMessage());
135 |             return null;
136 |         } catch (Exception e) {
137 |             e.printStackTrace();
138 |             logger.error(e.getMessage());
139 |             return null;
140 |         } finally {
141 |             //关闭流和socket
142 |             if (osClient != null) {
143 |                 try {
144 |                     osClient.close();
145 |                 } catch (IOException ignored) {
146 |                 }
147 |             }
148 |             if (isClient != null) {
149 |                 try {
150 |                     isClient.close();
151 |                 } catch (IOException ignored) {
152 |                 }
153 |             }
154 |             if (!inSocket.isClosed()) {
155 |                 try {
156 |                     inSocket.close();
157 |                 } catch (IOException ignored) {
158 |                 }
159 |             }
160 |             if (isTarget != null) {
161 |                 try {
162 |                     isTarget.close();
163 |                 } catch (IOException ignored) {
164 |                 }
165 |             }
166 |             if (osTarget != null) {
167 |                 try {
168 |                     osTarget.close();
169 |                 } catch (IOException ignored) {
170 |                 }
171 |             }
172 |             if (!targetSocket.isClosed()) {
173 |                 try {
174 |                     targetSocket.close();
175 |                 } catch (IOException ignored) {
176 |                 }
177 |             }
178 |         }
179 |         return null;
180 |     }
181 | 
182 |     /**
183 |      * 从一端读取数据，传输到另外一端
184 |      *
185 |      * @param isFrom 内容获取
186 |      * @param osOut  内容输出
187 |      * @throws IOException
188 |      */
189 |     private void readDataToOther(InputStream isFrom, OutputStream osOut) throws IOException {
190 |         byte[] buffer = new byte[1024 * 4];
191 |         int len = 0;
192 |         logger.debug("available:" + isFrom.available());
193 |         while ((len = isFrom.read(buffer)) != -1) {
194 |             osOut.write(buffer, 0, len);
195 |             osOut.flush();
196 |         }
197 |     }
198 | 
199 |     /**
200 |      * 从目标网站服务器读取数据，传输给使用代理服务器的客户端
201 |      *
202 |      * @param isTarget
203 |      * @param osClient
204 |      */
205 |     private void readFromTargetToClient(InputStream isTarget, OutputStream osClient) throws IOException {
206 | //        readDataToOther(isTarget,osClient);
207 |         byte[] buffer = new byte[1024 * 8];
208 |         int len = 0;
209 |         logger.debug("available:" + isTarget.available());
210 |         try {
211 |             while ((len = isTarget.read(buffer)) != -1) {
212 |                 logger.debug("从目标网站读取到数据长度：" + len);
213 |                 osClient.write(buffer, 0, len);
214 |                 osClient.flush();
215 |                 if (targetSocket.isOutputShutdown() || inSocket.isClosed()) {
216 |                     break;
217 |                 }
218 |             }
219 |         } catch (SocketTimeoutException e) {
220 |         }
221 |     }
222 | 
223 |     /**
224 |      * 发送剩余的内容到目标服务器
225 |      *
226 |      * @param isClient 内容输入端
227 |      * @param osTarget 内容接受端
228 |      */
229 |     private void transferDataToTarget(InputStream isClient, OutputStream osTarget) throws IOException {
230 | //        readDataToOther(isClient,osTarget);
231 | 
232 |         byte[] buffer = new byte[1024 * 4];
233 |         int len = 0;
234 |         logger.debug("available:" + isClient.available());
235 |         try {
236 |             while ((len = isClient.read(buffer)) != -1) {
237 |                 osTarget.write(buffer, 0, len);
238 |                 osTarget.flush();
239 |                 if (inSocket.isOutputShutdown() || targetSocket.isClosed()) {
240 |                     break;
241 |                 }
242 |             }
243 |         } catch (SocketTimeoutException e) {
244 | 
245 |         }
246 | 
247 |     }
248 | 
249 |     /**
250 |      * 错误的请求的报文
251 |      *
252 |      * @return 报文内容
253 |      */
254 |     private String badRequest() {
255 |         StringBuilder htmlBuilder = new StringBuilder();
256 |         htmlBuilder.append("<html>\r\n")
257 |                 .append("<head>\r\rn")
258 |                 .append("<title>Bad Request</title>\r\n")
259 |                 .append("</head>\r\n")
260 |                 .append("<body>Bad Request</body>\r\n")
261 |                 .append("</html>\r\n");
262 |         StringBuilder reporterBudiler = new StringBuilder();
263 |         reporterBudiler.append("HTTP/1.1 400 Bad Request\r\n")
264 |                 .append("Content-Type:text/html;charset=ISO-8859-1\r\n")
265 |                 .append("Content-Length:" + htmlBuilder.toString().length() + "\r\n")
266 |                 .append("\r\n")
267 |                 .append(htmlBuilder.toString());
268 |         return reporterBudiler.toString();
269 |     }
270 | 
271 | 
272 | }
273 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/bean/HttpHeader.java:
--------------------------------------------------------------------------------
 1 | package com.caspar.bean;
 2 | import java.util.List;
 3 | 
 4 | /**
 5 |  * 报文解析
 6 |  * Created by casparhuan on 2016/12/3.
 7 |  */
 8 | public final class HttpHeader {
 9 |     private final String requestLine;//请求头
10 |     private final List<String> headers ;//报文头部
11 | 
12 |     private final String host;
13 |     private final int port;
14 |     private final String method;
15 |     private final String requestURL;
16 | 
17 |     public static final String METHOD_GET = "GET";
18 |     public static final String METHOD_POST = "POST";
19 |     public static final String METHOD_CONNECTION = "CONNECT";
20 | 
21 | 
22 |     public HttpHeader(String host,int port,String method,String requestLine,String requestURL,List<String> headers){
23 |         this.headers = headers;
24 |         this.host = host;
25 |         this.port = port;
26 |         this.requestLine = requestLine;
27 |         this.method = method;
28 |         this.requestURL =requestURL;
29 |     }
30 | 
31 |     /**
32 |      * 返回整个报文头部
33 |      * @return
34 |      */
35 |     public String getHttpHedaer(){
36 |         StringBuilder sb = new StringBuilder();
37 |         sb.append(requestLine)
38 |                 .append("\r\n");
39 |         headers.forEach(val->{
40 |             sb.append(val).append("\r\n");
41 |         });
42 |         sb.append("\r\n");
43 | //        System.out.println("----------");
44 | //        System.out.println(sb.toString()+"post date:");
45 | //        System.out.println("----------");
46 |         return sb.toString();
47 |     }
48 | 
49 |     public List<String> getHeaders() {
50 |         return headers;
51 |     }
52 | 
53 |     public String getHost() {
54 |         return host;
55 |     }
56 | 
57 |     public int getPort() {
58 |         return port;
59 |     }
60 | 
61 |     public String getMethod() {
62 |         return method;
63 |     }
64 | 
65 |     public String getRequestLine() {
66 |         return requestLine;
67 |     }
68 | 
69 |     public String getRequestURL() {
70 |         return requestURL;
71 |     }
72 | 
73 |     @Override
74 |     public String toString() {
75 |         return "HttpHeader{" +
76 |                 "requestLine='" + requestLine + '\'' +
77 |                 ", headers=" + headers +
78 |                 ", host='" + host + '\'' +
79 |                 ", port=" + port +
80 |                 ", method='" + method + '\'' +
81 |                 ", requestURL='" + requestURL + '\'' +
82 |                 '}';
83 |     }
84 | }
85 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/exception/HttpHeaderConstructException.java:
--------------------------------------------------------------------------------
 1 | package com.caspar.exception;
 2 | 
 3 | /**
 4 |  * Created by casparhuan on 2016/12/4.
 5 |  */
 6 | public class HttpHeaderConstructException extends Exception{
 7 | 
 8 |     public HttpHeaderConstructException() {
 9 |         super();
10 |     }
11 | 
12 |     public HttpHeaderConstructException(String message) {
13 |         super(message);
14 |     }
15 | }
16 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/util/HttpClientUtil.java:
--------------------------------------------------------------------------------
  1 | package com.caspar.util;
  2 | 
  3 | import org.apache.http.HttpEntity;
  4 | import org.apache.http.NameValuePair;
  5 | import org.apache.http.client.ClientProtocolException;
  6 | import org.apache.http.client.config.RequestConfig;
  7 | import org.apache.http.client.entity.UrlEncodedFormEntity;
  8 | import org.apache.http.client.methods.*;
  9 | import org.apache.http.client.utils.URIBuilder;
 10 | import org.apache.http.impl.client.CloseableHttpClient;
 11 | import org.apache.http.impl.client.HttpClients;
 12 | import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 13 | import org.apache.http.message.BasicNameValuePair;
 14 | import org.apache.http.util.EntityUtils;
 15 | 
 16 | import java.io.IOException;
 17 | import java.io.UnsupportedEncodingException;
 18 | import java.net.URISyntaxException;
 19 | import java.util.ArrayList;
 20 | import java.util.Map;
 21 | 
 22 | 
 23 | public class HttpClientUtil {
 24 | 	private static PoolingHttpClientConnectionManager cm;
 25 |     private static String EMPTY_STR = "none";
 26 |     private static String UTF_8 = "UTF-8";
 27 |     public static String CURRENT = "";
 28 |     public static String CURRENT1 = "";
 29 |     public static int timeOut = 10;
 30 | 
 31 |     private static void init() {
 32 |         if (cm == null) {
 33 |             cm = new PoolingHttpClientConnectionManager();
 34 |             cm.setMaxTotal(2000);// 整个连接池最大连接数
 35 |             cm.setDefaultMaxPerRoute(500);// 每路由最大连接数，默认值是2
 36 |         }
 37 |     }
 38 | 
 39 |     /**
 40 |      * 通过连接池获取HttpClient
 41 |      * 
 42 |      * @return
 43 |      */
 44 |     public static CloseableHttpClient getHttpClient() {
 45 |         init();
 46 |         return HttpClients.custom().setConnectionManager(cm).build();
 47 |     }
 48 | 
 49 | 
 50 | 
 51 | //    public static void getSize(){
 52 | //        if(cm!=null){
 53 | //            System.out.println(cm.getDefaultConnectionConfig().getBufferSize());
 54 | //        }
 55 | //    }
 56 | 
 57 |     /**
 58 |      * 
 59 |      * @param url
 60 |      * @return
 61 |      */
 62 |     public static String httpGetRequest(String url) {
 63 |     	
 64 |     	
 65 |         HttpGet httpGet = new HttpGet(url);
 66 |         
 67 |         return getResult(httpGet);
 68 |     }
 69 | 
 70 |     /**
 71 |      * 获取get请求响应
 72 |      * @param url
 73 |      * @return
 74 |      * @throws IOException
 75 |      */
 76 | 
 77 |     public static CloseableHttpResponse get(String url) throws IOException {
 78 | 
 79 | 
 80 |         HttpGet httpGet = new HttpGet(url);
 81 | /*        SocketConfig socketConfig = SocketConfig.custom()
 82 |                 .setSoKeepAlive(false)
 83 |                 .setSoLinger(1)
 84 |                 .setSoReuseAddress(true)
 85 |                 .setSoTimeout(timeOut*100)
 86 |                 .setTcpNoDelay(true).build();*/
 87 |         RequestConfig requestConfig = RequestConfig.custom()
 88 |                 .setConnectTimeout(timeOut*100).build();
 89 |         httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36" );
 90 |         httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
 91 |         httpGet.setHeader( "Accept-Language","zh-CN,zh;q=0.8,en;q=0.6");
 92 |         httpGet.setConfig(requestConfig);
 93 |         return getHttpClient().execute(httpGet);
 94 | 
 95 |     }
 96 | 
 97 |     /**
 98 |      * 获取head请求响应
 99 |      * @param url
100 |      * @return
101 |      * @throws IOException
102 |      */
103 | 
104 | 
105 |     public static CloseableHttpResponse head(String url) throws IOException {
106 | 
107 | 
108 |         HttpHead httpHead = new HttpHead(url);
109 |         httpHead.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36" );
110 |         httpHead.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
111 |         httpHead.setHeader( "Accept-Language","zh-CN,zh;q=0.8,en;q=0.6");
112 |         RequestConfig requestConfig = RequestConfig.custom()
113 |                 .setConnectTimeout(timeOut*100).build();
114 |         httpHead.setConfig(requestConfig);
115 |         // response.getStatusLine().getStatusCode();
116 |         return getHttpClient().execute(httpHead);
117 | 
118 |     }
119 |     
120 |     public static String httpGetRequest1(String url,Map<String, Object> headers) {
121 |         HttpGet httpGet = new HttpGet(url);
122 |         for (Map.Entry<String, Object> param : headers.entrySet()) {
123 |             httpGet.addHeader(param.getKey(), String.valueOf(param.getValue()));
124 |         }
125 |         return getResult(httpGet);
126 |     }
127 |     
128 |     public static String httpHeadRequest(String url) {
129 |         HttpHead httpHead = new HttpHead(url);
130 |         return getResult(httpHead);
131 |     }
132 | 
133 |     public static String httpGetRequest(String url, Map<String, Object> params) throws URISyntaxException {
134 |     	
135 |     	
136 |     	// URIBuilder 实用类来简化请求 URL的创建和修改.
137 |         URIBuilder ub = new URIBuilder();
138 |         
139 |         //ub.setHost(url);
140 |         
141 |         //设置请求路径  e.g /index
142 |         ub.setPath(url);
143 | 
144 |         ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
145 |         
146 |     /*    for (NameValuePair nameValuePair : pairs) {
147 | 			System.out.println("参数打印"+nameValuePair.getName()+"="+nameValuePair.getValue());
148 | 		}*/
149 |         
150 |         //设置参数
151 |         ub.setParameters(pairs);
152 | 
153 |         HttpGet httpGet = new HttpGet(ub.build());
154 |         
155 |         
156 |         //打印url
157 |         //System.out.println(httpGet.getURI());
158 |         CURRENT=httpGet.getURI().toString();
159 |         return getResult(httpGet);
160 |     }
161 | 
162 |     public static String httpGetRequest(String url, Map<String, Object> headers, Map<String, Object> params)
163 |             throws URISyntaxException {
164 |         URIBuilder ub = new URIBuilder();
165 |         ub.setPath(url);
166 | 
167 |         ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
168 |         ub.setParameters(pairs);
169 | 
170 |         HttpGet httpGet = new HttpGet(ub.build());
171 |         for (Map.Entry<String, Object> param : headers.entrySet()) {
172 |             httpGet.addHeader(param.getKey(), String.valueOf(param.getValue()));
173 |         }
174 |         return getResult(httpGet);
175 |     }
176 | 
177 |     public static String httpPostRequest(String url) {
178 |         HttpPost httpPost = new HttpPost(url);
179 |         return getResult(httpPost);
180 |     }
181 |     
182 |     public static String httpPostRequest1(String url,Map<String, Object> headers) {
183 |         HttpPost httpPost = new HttpPost(url);
184 |         for (Map.Entry<String, Object> param : headers.entrySet()) {
185 |             httpPost.addHeader(param.getKey(), String.valueOf(param.getValue()));
186 |         }
187 |         return getResult(httpPost);
188 |     }
189 | 
190 |     public static String httpPostRequest(String url, Map<String, Object> params) throws UnsupportedEncodingException {
191 |         HttpPost httpPost = new HttpPost(url);
192 |         ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
193 |         httpPost.setEntity(new UrlEncodedFormEntity(pairs, UTF_8));
194 |         return getResult(httpPost);
195 |     }
196 | 
197 |     public static String httpPostRequest(String url, Map<String, Object> headers, Map<String, Object> params)
198 |             throws UnsupportedEncodingException {
199 |         HttpPost httpPost = new HttpPost(url);
200 | 
201 |         for (Map.Entry<String, Object> param : headers.entrySet()) {
202 |             httpPost.addHeader(param.getKey(), String.valueOf(param.getValue()));
203 |         }
204 | 
205 |         ArrayList<NameValuePair> pairs = covertParams2NVPS(params);
206 |         httpPost.setEntity(new UrlEncodedFormEntity(pairs, UTF_8));
207 | 
208 |         return getResult(httpPost);
209 |     }
210 | 
211 |     private static ArrayList<NameValuePair> covertParams2NVPS(Map<String, Object> params) {
212 |         ArrayList<NameValuePair> pairs = new ArrayList<NameValuePair>();
213 |         
214 |         
215 |         for (Map.Entry<String, Object> param : params.entrySet()) {
216 |         	if(param.getKey().equals("avars[1][]")) {
217 |         		pairs.add(new BasicNameValuePair("vars[1][]", String.valueOf(param.getValue())));
218 |         	}else {
219 |         		pairs.add(new BasicNameValuePair(param.getKey(), String.valueOf(param.getValue())));
220 |         	}
221 |             
222 |         }
223 | 
224 |         return pairs;
225 |     }
226 | 
227 |     /**
228 |      * 处理Http请求
229 |      * 
230 |      * @param request
231 |      * @return
232 |      */
233 |     private static String getResult(HttpRequestBase request) {
234 |         // CloseableHttpClient httpClient = HttpClients.createDefault();
235 |         CloseableHttpClient httpClient = getHttpClient();
236 |         try {
237 |             CloseableHttpResponse response = httpClient.execute(request);
238 |             // response.getStatusLine().getStatusCode();
239 |             
240 |             //响应实例
241 |             HttpEntity entity = response.getEntity();
242 | 
243 |             if (entity != null) {
244 |                 // long len = entity.getContentLength();// -1 表示长度未知
245 |                 String result = EntityUtils.toString(entity,"utf-8");
246 |                 response.close();
247 |                 // httpClient.close();
248 |                 //System.out.println(result);
249 |                 return result;
250 |             }
251 |         } catch (ClientProtocolException e) {
252 |         	e.printStackTrace();
253 |         } catch (IOException e) {
254 |         	e.printStackTrace();
255 |         }
256 | 
257 |         return EMPTY_STR;
258 |     }
259 |     
260 | 
261 | }
262 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/util/HttpHeaderBuilder.java:
--------------------------------------------------------------------------------
  1 | package com.caspar.util;
  2 | 
  3 | import com.caspar.bean.HttpHeader;
  4 | import com.caspar.exception.HttpHeaderConstructException;
  5 | 
  6 | import java.io.IOException;
  7 | import java.io.InputStream;
  8 | import java.util.ArrayList;
  9 | import java.util.Arrays;
 10 | import java.util.List;
 11 | 
 12 | /**
 13 |  * Created by casparhuan on 2016/12/3.
 14 |  */
 15 | public class HttpHeaderBuilder {
 16 |     private String requestLine;//请求头
 17 |     private List<String> headers ;//报文头部
 18 |     private String host;
 19 |     private int port;
 20 |     private String method;
 21 |     private String requestURL;
 22 |     private String[] leagalMehod = {"GET","POST","CONNECT"};
 23 | 
 24 |     private static final String METHOD_GET = HttpHeader.METHOD_GET;
 25 |     private static final String METHOD_POST = HttpHeader.METHOD_POST;
 26 |     private static final String METHOD_CONNECTION = HttpHeader.METHOD_CONNECTION;
 27 | 
 28 |     private final static int  MAX_REQUEST_LINE = 2048;
 29 |     private final static int  MAX_HEADER_LENGTH = 16;
 30 | 
 31 |     private final InputStream is;
 32 | 
 33 |     public HttpHeaderBuilder(InputStream is){
 34 |         this.is = is;
 35 |         this.method = "";
 36 |         this.port = 0;
 37 |         this.headers = new ArrayList<String>();
 38 |         this.requestLine ="";
 39 |         this.host = "";
 40 |     }
 41 | 
 42 |     public HttpHeader build() throws HttpHeaderConstructException {
 43 |         try {
 44 |             //1. 读取第一行如：GET http://baidu.com/ HTTP/1.1
 45 |             //1.1 读取请求头
 46 |             StringBuilder stringBuilder = new StringBuilder();
 47 |             int readTemp = -1;
 48 |             char readTempChar = 0;
 49 |             int requestLineLength = 0;
 50 |             while ((readTemp = is.read()) != -1) {
 51 |                 readTempChar = (char) readTemp;
 52 |                 if (readTempChar == '\n') {
 53 |                     break;
 54 |                 }
 55 |                 requestLineLength++;
 56 |                 stringBuilder.append(readTempChar);
 57 |                 if (requestLineLength > MAX_REQUEST_LINE) {
 58 |                     throw new HttpHeaderConstructException("read requestLine error");
 59 |                 }
 60 |             }
 61 |             requestLine = stringBuilder.toString().replace("\r", "");
 62 |             //1.2 解析请求头
 63 |             parseRequestLine(requestLine);
 64 | 
 65 |             //2. 获取header
 66 |             boolean isNotHttpEntity = false;//是否是到了实体主体部分
 67 |             while (!isNotHttpEntity) {
 68 |                 //2.1 清空stringbuilder
 69 |                 stringBuilder.setLength(0);
 70 |                 //2.2 循环读取header
 71 |                 while ((readTemp = is.read()) != -1) {
 72 |                     readTempChar = (char) readTemp;
 73 |                     if (readTempChar == '\n') {
 74 |                         if (stringBuilder.toString().equals("\r")) {
 75 |                             isNotHttpEntity = true;//已经到了实体主体部分了
 76 |                             break;
 77 |                         }
 78 |                         break;
 79 |                     }
 80 |                     stringBuilder.append(readTempChar);
 81 |                 }
 82 |                 //2.3 添加头部
 83 |                 String header = stringBuilder.toString().replace("\r", "");
 84 |                 if(header.trim().length()==0){
 85 |                     break;
 86 |                 }
 87 |                 headers.add(header);
 88 |                 //2.3 检查是否是host 的header(如：   Host: baidu.com)
 89 |                 String[] temps = header.split(":");
 90 |                 if (temps.length != 0 && temps[0].equalsIgnoreCase("host")) {
 91 |                     host = temps[1].trim();
 92 |                     if (temps.length > 3) {
 93 |                         throw new HttpHeaderConstructException("read host Hedaer error");
 94 |                     }
 95 |                     if (temps.length == 2) {
 96 |                         if (METHOD_CONNECTION.equals(temps)) {
 97 |                             //HTTPS 默认端口
 98 |                             port = 443;
 99 |                         } else {
100 |                             //HTTP 默认端口
101 |                             port = 80;
102 |                         }
103 |                     } else {
104 |                         try {
105 |                             port = Integer.parseInt(temps[2].trim());
106 |                         } catch (NumberFormatException e) {
107 |                             e.printStackTrace();
108 |                             throw new HttpHeaderConstructException("read host port Hedaer error");
109 |                         }
110 |                     }
111 |                 }
112 | 
113 |                 if (headers.size() >= MAX_HEADER_LENGTH) {
114 |                     throw new HttpHeaderConstructException("headers too long");
115 |                 }
116 | 
117 |             }
118 |         }catch (IOException ioe){
119 |             ioe.printStackTrace();
120 |             throw new HttpHeaderConstructException(ioe.getMessage());
121 |         }finally {
122 |             return new HttpHeader(host,port,method,requestLine,requestURL,headers);
123 |         }
124 |     }
125 | 
126 |     /**
127 |      * 校验method
128 |      * @param method
129 |      * @return
130 |      */
131 |     private boolean validateMethod(String method){
132 |         if(method ==null || method.trim().length()==0){
133 |             return false;
134 |         }
135 |         for (int i = 0; i < leagalMehod.length; i++) {
136 |             if(method.equals(leagalMehod[i])){
137 |                 return true;
138 |             }
139 |         }
140 |         return false;
141 |     }
142 | 
143 |     /**
144 |      * 解析请求头
145 |      * @param requestLine 如：GET http://baidu.com/ HTTP/1.1
146 |      */
147 |     private void parseRequestLine(String requestLine) {
148 |         int index1,index2;
149 |         index1 = requestLine.indexOf(' ');
150 |         if( index1 != -1 ){
151 |             method = requestLine.substring(0,index1);
152 |             if(!validateMethod(method)){
153 |                 method = "";
154 |                 return;
155 |             }
156 |             index2 = requestLine.substring(index1+1).indexOf(' ');
157 |             if(index2 > index1){
158 |                 requestURL = requestLine.substring(index1+1,index1+1+index2);
159 |             }
160 |         }
161 |     }
162 | 
163 |     public String getRequestLine() {
164 |         return requestLine;
165 |     }
166 | 
167 |     public List<String> getHeaders() {
168 |         return headers;
169 |     }
170 | 
171 |     public String getHost() {
172 |         return host;
173 |     }
174 | 
175 |     public int getPort() {
176 |         return port;
177 |     }
178 | 
179 |     public String getMethod() {
180 |         return method;
181 |     }
182 | 
183 |     public String getRequestURL() {
184 |         return requestURL;
185 |     }
186 | 
187 |     public String[] getLeagalMehod() {
188 |         return leagalMehod;
189 |     }
190 | 
191 |     @Override
192 |     public String toString() {
193 |         return "HttpHeaderBuilder{" +
194 |                 "leagalMehod=" + Arrays.toString(leagalMehod) +
195 |                 ", requestLine='" + requestLine + '\'' +
196 |                 ", headers=" + headers +
197 |                 ", host='" + host + '\'' +
198 |                 ", port=" + port +
199 |                 ", method='" + method + '\'' +
200 |                 ", requestURL='" + requestURL + '\'' +
201 |                 '}';
202 |     }
203 | }
204 | 


--------------------------------------------------------------------------------
/src/main/java/com/caspar/util/ProxyUtil.java:
--------------------------------------------------------------------------------
 1 | package com.caspar.util;
 2 | 
 3 | import com.alibaba.fastjson.JSON;
 4 | import com.caspar.ProxyServer;
 5 | import org.slf4j.Logger;
 6 | import org.slf4j.LoggerFactory;
 7 | 
 8 | import java.net.InetSocketAddress;
 9 | import java.net.Proxy;
10 | import java.util.Map;
11 | 
12 | /**
13 |  * @Description: 二级代理工具类
14 |  * @author: safe6
15 |  * @date: 2020年07月20日 下午2:57
16 |  */
17 | public class ProxyUtil {
18 | 
19 |     private static Logger logger = LoggerFactory.getLogger(ProxyUtil.class.getName());
20 | 
21 | 
22 |     /**
23 |      * 获取二级代理
24 |      * @return
25 |      */
26 |     public static Proxy getProxy(){
27 | 
28 |         Proxy proxy = null;
29 |         try {
30 |             //String url = "http://118.24.52.95/get/";
31 |             String url = "http://localhost:5010/get/";
32 | 
33 |             String request = HttpClientUtil.httpGetRequest(url);
34 | 
35 |             if (!request.contains("proxy")){
36 |                 logger.error("获取代理失败,重试中。。。");
37 |                 getProxy();
38 |             }
39 | 
40 |             Map<String,Object> map = JSON.parseObject(request,Map.class);
41 |             String ip = map.get("proxy").toString();
42 |             logger.info("使用代理:"+ip);
43 |             String[] split = ip.split(":");
44 | 
45 |             proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(split[0], Integer.parseInt(split[1])));
46 | 
47 |         } catch (Exception e) {
48 |             e.printStackTrace();
49 |             getProxy();
50 |         }
51 | 
52 |         return proxy;
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | 
 3 | <configuration debug="true" scan="true" scanPeriod="30 seconds">
 4 | 
 5 |     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 6 |         <!-- encoders are  by default assigned the type
 7 |              ch.qos.logback.classic.encoder.PatternLayoutEncoder -->
 8 |         <encoder>
 9 |             <pattern>%d{yyyy-MM-dd HH:mm:ss} [%p][%c][%M][%L]-> %m%n</pattern>
10 |             <!-- 常用的Pattern变量,大家可打开该pattern进行输出观察 -->
11 |             <!--
12 |               <pattern>
13 |                   %d{yyyy-MM-dd HH:mm:ss} [%level] - %msg%n
14 |                   Logger: %logger
15 |                   Class: %class
16 |                   File: %file
17 |                   Caller: %caller
18 |                   Line: %line
19 |                   Message: %m
20 |                   Method: %M
21 |                   Relative: %relative
22 |                   Thread: %thread
23 |                   Exception: %ex
24 |                   xException: %xEx
25 |                   nopException: %nopex
26 |                   rException: %rEx
27 |                   Marker: %marker
28 |                   %n
29 | 
30 |               </pattern>
31 |                -->
32 |         </encoder>
33 |     </appender>
34 | 
35 | 
36 |     <root level="debug">
37 |         <appender-ref ref="STDOUT" />
38 |     </root>
39 | </configuration>


--------------------------------------------------------------------------------
/src/test/java/com/caspar/someThing/TestChar.java:
--------------------------------------------------------------------------------
 1 | package com.caspar.someThing;
 2 | 
 3 | /**
 4 |  * Created by casparhuan on 2016/12/3.
 5 |  */
 6 | public class TestChar {
 7 |     public static void main(String[] args) {
 8 |         char ttt = (char) -1;
 9 |         if(ttt  == (char)-1){
10 |             System.out.println("xxxx");
11 |         }
12 |         if(ttt  == -1){
13 |             System.out.println("yyy");
14 |         }
15 |         System.out.println("ttt:"+ttt);
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/src/test/java/com/caspar/someThing/TestSocket.java:
--------------------------------------------------------------------------------
 1 | package com.caspar.someThing;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | import java.io.IOException;
 6 | import java.io.InputStream;
 7 | import java.io.OutputStream;
 8 | import java.net.Socket;
 9 | 
10 | /**
11 |  * Created by casparhuan on 2016/12/4.
12 |  */
13 | public class TestSocket {
14 |     @Test
15 |     public void testViewBaidu() throws IOException {
16 | //        System.getProperties().put("socksProxySet", "true");
17 | //        System.getProperties().put("socksProxyHost", "127.0.0.1");
18 | //        System.getProperties().put("socksProxyPort","8882");
19 | 
20 | 
21 |         StringBuilder rp = new StringBuilder();
22 |         rp.append("GET http://www.ifeng.com/ HTTP/1.1\r\n")
23 |                 .append(
24 |                         //"User-Agent: Fiddler\r\n" +
25 |                         "Connection: keep-alive\r\n" +
26 |                         "Content-Length: 0\r\n" +
27 |                         "Content-Type: text/plain charset=utf-8\r\n" +
28 |                         "Host: www.ifeng.com\r\n" +
29 |                         "Proxy-Connection: Keep-Alive\r\n" +
30 |                         "User-Agent: Apache-HttpClient/4.5.2 (Java/1.8.0_71)\r\n")
31 |                 .append("\r\n");
32 |         System.out.println("------------");
33 |         System.out.println(rp.toString());
34 |         System.out.println("------------");
35 |         Socket socket = new Socket("127.0.0.1",8002);
36 |         InputStream in = socket.getInputStream();
37 |         OutputStream out = socket.getOutputStream();
38 |         out.write(rp.toString().getBytes());
39 |         out.flush();
40 | 
41 |         byte[] buffer = new byte[1024*100];
42 |         int len = 0;
43 |         StringBuilder html = new StringBuilder();
44 |         while( (len = in.read(buffer))!=-1){
45 | //            System.out.println("读取："+len);
46 | //            System.out.println(new String(buffer,"utf-8"));
47 | //            if(in.available()==0){
48 | //                break;
49 | //            }
50 |             html.append(new String(buffer,"utf-8"));
51 |         }
52 |         System.out.println(html);
53 | 
54 | 
55 |     }
56 | }
57 | 


--------------------------------------------------------------------------------
/src/test/java/com/caspar/util/TestHttpHeaderBuilder.java:
--------------------------------------------------------------------------------
 1 | package com.caspar.util;
 2 | 
 3 | import org.junit.Assert;
 4 | import org.junit.Test;
 5 | 
 6 | import java.lang.reflect.InvocationTargetException;
 7 | import java.lang.reflect.Method;
 8 | 
 9 | /**
10 |  * Created by casparhuan on 2016/12/3.
11 |  */
12 | public class TestHttpHeaderBuilder {
13 | 
14 |     @Test
15 |     public void testParseRequestLine() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
16 |         HttpHeaderBuilder httpHeaderBuilder = new HttpHeaderBuilder(null);
17 |         Method method = HttpHeaderBuilder.class.getDeclaredMethod("parseRequestLine", String.class);
18 |         method.setAccessible(true);
19 |         method.invoke(httpHeaderBuilder, "GET http://baidu.com/ HTTP/1.1");
20 |         Assert.assertEquals("method eqauls","GET",httpHeaderBuilder.getMethod());
21 |         Assert.assertEquals("requestURL","http://baidu.com/",httpHeaderBuilder.getRequestURL());
22 |         System.out.println(httpHeaderBuilder.toString());
23 |     }
24 | 
25 | }
26 | 


--------------------------------------------------------------------------------