├── .gitignore
├── .travis.yml
├── ProxyPool.iml
├── README.md
├── pom.xml
└── src
├── main
├── java
│ └── com
│ │ └── ruyuapp
│ │ ├── Main.java
│ │ ├── proxy
│ │ ├── HttpProxy.java
│ │ └── ProxyPool.java
│ │ └── util
│ │ ├── HttpStatus.java
│ │ ├── IpUtils.java
│ │ └── ProxyPersistUtils.java
└── resources
│ └── log4j.properties
└── test
└── java
└── com
└── ruyuapp
└── util
└── IpUtilsTest.java
/.gitignore:
--------------------------------------------------------------------------------
1 | # Created by .ignore support plugin (hsz.mobi)
2 | /.idea/
3 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: java
2 | jdk:
3 | - oraclejdk7
--------------------------------------------------------------------------------
/ProxyPool.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # ProxyPool
2 |
3 | [](https://travis-ci.org/letcheng/ProxyPool)
4 | [](https://jitpack.io/#letcheng/ProxyPool)
5 |
6 | 针对反爬虫问题的自动代理池组件
7 |
8 | ### 特色
9 |
10 | * 支持 Proxy 自动持久化，一次加入，永久可使用
11 | * 自动将不可用的 Proxy 移出代理池
12 | * 线程安全，支持多线程同时使用代理池
13 | * 优先选择响应速度快的 Proxy
14 | * 出现 403 等状态码时，自动降低该 Proxy 的访问频率
15 |
16 | ### 使用
17 |
18 | 1.添加 Maven 库
19 | ```
20 |
21 |
22 | jitpack.io
23 | https://jitpack.io
24 |
25 |
26 | ```
27 |
28 | ```
29 |
30 | com.github.letcheng
31 | ProxyPool
32 | x.x
33 |
34 | ```
35 |
36 | 2.采取 add(init) -> borrow -> reback 的方式进行使用
37 |
38 | ```java
39 | ProxyPool proxyPool = new ProxyPool();
40 |
41 | proxyPool.add("203.171.230.230", 80);
42 | proxyPool.add("121.9.221.188", 80);
43 |
44 | HttpProxy httpProxy = proxyPool.borrow(); // 从 ProxyPool 中获取一个Proxy
45 |
46 | proxyPool.reback(httpProxy, HttpStatus.SC_OK); // 使用完成之后，归还 Proxy,并将请求结果的 http 状态码一起传入
47 |
48 | proxyPool.allProxyStatus(); // 可以获取 ProxyPool 中所有 Proxy 的当前状态
49 | ```
50 |
51 | 3.enjoy!
52 |
53 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 | 4.0.0
6 |
7 | com.github.letcheng
8 | ProxyPool
9 | 1.0-SNAPSHOT
10 |
11 |
12 |
13 | log4j
14 | log4j
15 | 1.2.17
16 |
17 |
18 | org.slf4j
19 | slf4j-api
20 | 1.7.21
21 |
22 |
23 | org.slf4j
24 | slf4j-log4j12
25 | 1.7.21
26 |
27 |
28 | commons-io
29 | commons-io
30 | 2.4
31 |
32 |
33 | junit
34 | junit
35 | 4.12
36 | test
37 |
38 |
39 |
40 |
--------------------------------------------------------------------------------
/src/main/java/com/ruyuapp/Main.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.ruyuapp;
5 |
6 | import com.ruyuapp.proxy.HttpProxy;
7 | import com.ruyuapp.proxy.ProxyPool;
8 | import com.ruyuapp.util.HttpStatus;
9 |
10 | import java.io.IOException;
11 | import java.net.*;
12 | import java.util.concurrent.TimeUnit;
13 |
14 | /**
15 | * @author letcheng
16 | * @version create at 2016年3月28日 15:32
17 | */
18 | public class Main {
19 |
20 | /**
21 | * 为了阻塞主线程，不在junit中测试
22 | *
23 | * @param args
24 | */
25 | public static void main(String args[]) {
26 |
27 | ProxyPool proxyPool = new ProxyPool();
28 |
29 | proxyPool.add("203.171.230.230", 80);
30 | proxyPool.add("121.9.221.188", 80);
31 |
32 | HttpProxy httpProxy = proxyPool.borrow(); // 从 ProxyPool 中获取一个Proxy
33 |
34 | URL url = null;
35 | try {
36 | url = new URL("http://www.ruyuapp.com");
37 | HttpURLConnection uc = (HttpURLConnection)url.openConnection(httpProxy.getProxy());
38 | System.out.println(uc.getResponseCode());
39 | uc.connect();
40 | } catch (MalformedURLException e) {
41 | e.printStackTrace();
42 | } catch (IOException e) {
43 | e.printStackTrace();
44 | }
45 |
46 | proxyPool.reback(httpProxy, HttpStatus.SC_OK); // 使用完成之后，归还 Proxy,并将请求结果的 http 状态码一起传入
47 |
48 | proxyPool.allProxyStatus(); // 可以获取 ProxyPool 中所有 Proxy 的当前状态
49 |
50 |
51 | }
52 |
53 | }
54 |
--------------------------------------------------------------------------------
/src/main/java/com/ruyuapp/proxy/HttpProxy.java:
--------------------------------------------------------------------------------
1 | package com.ruyuapp.proxy;
2 |
3 | import com.ruyuapp.util.HttpStatus;
4 | import com.ruyuapp.util.IpUtils;
5 | import org.slf4j.Logger;
6 | import org.slf4j.LoggerFactory;
7 |
8 | import java.io.IOException;
9 | import java.io.Serializable;
10 | import java.net.InetAddress;
11 | import java.net.InetSocketAddress;
12 | import java.net.Proxy;
13 | import java.net.Socket;
14 | import java.util.HashMap;
15 | import java.util.Map;
16 | import java.util.concurrent.Delayed;
17 | import java.util.concurrent.TimeUnit;
18 |
19 | /**
20 | * @author letcheng
21 | * @version create at 2016年3月27日 10:47
22 | */
23 | public class HttpProxy implements Delayed {
24 |
25 | private final static Logger logger = LoggerFactory.getLogger(HttpProxy.class);
26 |
27 | public final static int DEFAULT_REUSE_TIME_INTERVAL = 1500;// ms，从一次请求结束到再次可以请求的默认时间间隔
28 | public final static int FAIL_REVIVE_TIME_INTERVAL = 2 * 60 * 60 * 1000; //ms,请求失败，重试的时间间隔
29 |
30 | private Proxy proxy;
31 | private InetAddress localAddr;
32 |
33 | private int reuseTimeInterval = 0;
34 | private Long canReuseTime = 0L; // 当前Proxy可重用的时间,纳秒
35 |
36 | private int failedNum = 0;
37 | private int borrowNum = 0;
38 |
39 | private Map countErrorStatus = new HashMap();
40 |
41 | public HttpProxy(String address, int port) {
42 | this(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(address, port)), 0, 0, DEFAULT_REUSE_TIME_INTERVAL);
43 | }
44 |
45 | public HttpProxy(Proxy proxy) {
46 | this(proxy, 0, 0, DEFAULT_REUSE_TIME_INTERVAL);
47 | }
48 |
49 | public HttpProxy(Proxy proxy, int borrowNum, int failedNum, int reuseTimeInterval) {
50 | this.localAddr = IpUtils.getLocalAddr(); // 获取当前机器的ip地址
51 | if (localAddr == null) {
52 | logger.error("cannot get local IP!");
53 | System.exit(0);
54 | }
55 | this.proxy = proxy;
56 | this.canReuseTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(reuseTimeInterval, TimeUnit.MILLISECONDS);
57 | }
58 |
59 | /**
60 | * 检查本地机器和Proxy之间的连通性
61 | *
62 | * @return
63 | */
64 | public boolean check() {
65 | boolean isReachable = false;
66 | Socket socket = null;
67 | try {
68 | socket = new Socket();
69 | socket.bind(new InetSocketAddress(localAddr, 0));
70 | socket.connect(proxy.address(), 3000);
71 | isReachable = true;
72 | } catch (Exception e) {
73 | logger.error("bad proxy >>>" + this.proxy.toString());
74 | } finally {
75 | if (socket != null) {
76 | try {
77 | socket.close();
78 | } catch (IOException e) {
79 | e.printStackTrace();
80 | }
81 | }
82 | }
83 | return isReachable;
84 | }
85 |
86 | public void success() {
87 | this.failedNum = 0; //将 failNum 清零
88 | countErrorStatus.clear();
89 | }
90 |
91 | /**
92 | * 代理失败，记录相应的错误状态码
93 | *
94 | * @param httpStatus
95 | */
96 | public void fail(HttpStatus httpStatus) {
97 | if (countErrorStatus.containsKey(httpStatus)) {
98 | countErrorStatus.put(httpStatus, countErrorStatus.get(httpStatus) + 1);
99 | } else {
100 | countErrorStatus.put(httpStatus, 1);
101 | }
102 | this.failedNum++;
103 | }
104 |
105 | /**
106 | * 输出错误请求的日志
107 | *
108 | * @return
109 | */
110 | public String countErrorStatus() {
111 | StringBuilder stringBuilder = new StringBuilder();
112 | for (Map.Entry entry : countErrorStatus.entrySet()) {
113 | stringBuilder.append(entry.getKey().name()).append("_").append(entry.getKey().getCode()).append("->").append(entry.getValue());
114 | }
115 | return stringBuilder.toString();
116 | }
117 |
118 | /**
119 | * 对请求进行计数
120 | */
121 | public void borrow() {
122 | this.borrowNum++;
123 | }
124 |
125 | public int getFailedNum() {
126 | return failedNum;
127 | }
128 |
129 | public int getBorrowNum() {
130 | return borrowNum;
131 | }
132 |
133 | public void setReuseTimeInterval(int reuseTimeInterval) {
134 | this.reuseTimeInterval = reuseTimeInterval;
135 | this.canReuseTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(reuseTimeInterval, TimeUnit.MILLISECONDS);
136 | }
137 |
138 | public int getReuseTimeInterval() {
139 | return this.reuseTimeInterval;
140 | }
141 |
142 | public Proxy getProxy() {
143 | return proxy;
144 | }
145 |
146 | public void setProxy(Proxy proxy) {
147 | this.proxy = proxy;
148 | }
149 |
150 | public int compareTo(Delayed o) {
151 | HttpProxy that = (HttpProxy) o;
152 | return canReuseTime > that.canReuseTime ? 1 : (canReuseTime < that.canReuseTime ? -1 : 0);
153 | }
154 |
155 | public long getDelay(TimeUnit unit) {
156 | return unit.convert(canReuseTime - System.nanoTime(), TimeUnit.NANOSECONDS);
157 | }
158 |
159 | @Override
160 | public String toString() {
161 | return this.proxy.toString()
162 | + ">>> 使用:" + borrowNum + "次 "
163 | + ">>> 连续失败:" + failedNum + "次"
164 | + ">>> 距离下次可用:" + TimeUnit.MILLISECONDS.convert(canReuseTime > System.nanoTime() ? canReuseTime - System.nanoTime() : 0, TimeUnit.NANOSECONDS) + " ms后";
165 | }
166 |
167 | public String getKey() {
168 | InetSocketAddress address = (InetSocketAddress) proxy.address();
169 | return address.getAddress().getHostAddress() + ":" + address.getPort();
170 | }
171 | }
172 |
--------------------------------------------------------------------------------
/src/main/java/com/ruyuapp/proxy/ProxyPool.java:
--------------------------------------------------------------------------------
1 | package com.ruyuapp.proxy;
2 |
3 | import com.ruyuapp.util.HttpStatus;
4 | import com.ruyuapp.util.ProxyPersistUtils;
5 | import org.slf4j.Logger;
6 | import org.slf4j.LoggerFactory;
7 |
8 | import java.util.*;
9 | import java.util.Map.Entry;
10 | import java.util.concurrent.*;
11 |
12 | /**
13 | * @author letcheng
14 | * @version create at 2016年3月27日 10:47
15 | */
16 | public class ProxyPool {
17 |
18 | private Logger logger = LoggerFactory.getLogger(ProxyPool.class);
19 |
20 | private BlockingQueue idleQueue = new DelayQueue(); // 存储空闲的Proxy
21 | private Map totalQueue = new ConcurrentHashMap(); // 存储所有的Proxy
22 |
23 | public ProxyPool() {
24 | final ProxyPersistUtils proxyAutoSave = new ProxyPersistUtils();
25 | // 读取上次的Proxy记录
26 | this.totalQueue = proxyAutoSave.read();
27 | for(Map.Entry entry : totalQueue.entrySet()){
28 | this.idleQueue.add(entry.getValue());
29 | }
30 | Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(new Runnable() {
31 | public void run() {
32 | proxyAutoSave.save(totalQueue);
33 | }
34 | }, 0, 5, TimeUnit.SECONDS); //10分钟执行保存Proxy的操作
35 | }
36 |
37 | /**
38 | * 添加Proxy
39 | *
40 | * @param httpProxies
41 | */
42 | public void add(HttpProxy... httpProxies) {
43 | for (HttpProxy httpProxy : httpProxies) {
44 | System.out.println(httpProxy.getKey());
45 | if (totalQueue.containsKey(httpProxy.getKey())) {
46 | continue;
47 | }
48 | if (httpProxy.check()) {
49 | httpProxy.success();
50 | idleQueue.add(httpProxy);
51 | totalQueue.put(httpProxy.getKey(), httpProxy);
52 | }
53 | }
54 | }
55 |
56 | public void add(String address, int port) {
57 | this.add(new HttpProxy(address, port));
58 | }
59 |
60 | /**
61 | * 得到Proxy
62 | *
63 | * @return
64 | */
65 | public HttpProxy borrow() {
66 | HttpProxy httpProxy = null;
67 | try {
68 | Long time = System.currentTimeMillis();
69 | httpProxy = idleQueue.take();
70 | double costTime = (System.currentTimeMillis() - time) / 1000.0;
71 | logger.info("get proxy time >>>> " + costTime);
72 |
73 | HttpProxy p = totalQueue.get(httpProxy.getKey());
74 | p.borrow();
75 | } catch (InterruptedException e) {
76 | e.printStackTrace();
77 | }
78 | if (httpProxy == null) {
79 | throw new NoSuchElementException();
80 | }
81 | return httpProxy;
82 | }
83 |
84 | /**
85 | * 反馈 Proxy
86 | *
87 | * @param httpProxy
88 | * @param httpStatus
89 | */
90 | public void reback(HttpProxy httpProxy, HttpStatus httpStatus) {
91 | switch (httpStatus) {
92 | case SC_OK:
93 | httpProxy.success();
94 | httpProxy.setReuseTimeInterval(HttpProxy.DEFAULT_REUSE_TIME_INTERVAL);
95 | break;
96 | case SC_FORBIDDEN:
97 | httpProxy.fail(httpStatus);
98 | httpProxy.setReuseTimeInterval(HttpProxy.DEFAULT_REUSE_TIME_INTERVAL * httpProxy.getFailedNum()); // 被网站禁止，调节更长时间的访问频率
99 | logger.info(httpProxy.getProxy() + " >>>> reuseTimeInterval is >>>> " + TimeUnit.SECONDS.convert(httpProxy.getReuseTimeInterval(), TimeUnit.MILLISECONDS));
100 | break;
101 | default:
102 | httpProxy.fail(httpStatus);
103 | break;
104 | }
105 | if (httpProxy.getFailedNum() > 20) { // 失败超过 20 次，移除代理池队列
106 | httpProxy.setReuseTimeInterval(HttpProxy.FAIL_REVIVE_TIME_INTERVAL);
107 | logger.error("remove proxy >>>> " + httpProxy.getProxy() + ">>>>" + httpProxy.countErrorStatus() + " >>>> remain proxy >>>> " + idleQueue.size());
108 | return;
109 | }
110 | if (httpProxy.getFailedNum() > 0 && httpProxy.getFailedNum() % 5 == 0) { //失败超过 5次，10次，15次，检查本机与Proxy的连通性
111 | if (!httpProxy.check()) {
112 | httpProxy.setReuseTimeInterval(HttpProxy.FAIL_REVIVE_TIME_INTERVAL);
113 | logger.error("remove proxy >>>> " + httpProxy.getProxy() + ">>>>" + httpProxy.countErrorStatus() + " >>>> remain proxy >>>> " + idleQueue.size());
114 | return;
115 | }
116 | }
117 | try {
118 | idleQueue.put(httpProxy);
119 | } catch (InterruptedException e) {
120 | e.printStackTrace();
121 | }
122 | }
123 |
124 | public void allProxyStatus() {
125 | String re = "all proxy info >>>> \n";
126 | for (Entry entry : totalQueue.entrySet()) {
127 | re += entry.getValue().toString() + "\n";
128 | }
129 | logger.info(re);
130 | }
131 |
132 | /**
133 | * 获取当前空闲的Proxy
134 | *
135 | * @return
136 | */
137 | public int getIdleNum() {
138 | return idleQueue.size();
139 | }
140 |
141 | }
142 |
--------------------------------------------------------------------------------
/src/main/java/com/ruyuapp/util/HttpStatus.java:
--------------------------------------------------------------------------------
1 | package com.ruyuapp.util;
2 |
3 | /**
4 | * @author letcheng
5 | * @version create at 2016年3月27日 10:47
6 | */
7 | public enum HttpStatus {
8 | SC_CONTINUE(100),
9 | SC_SWITCHING_PROTOCOLS(101),
10 | SC_PROCESSING(102),
11 | SC_OK(200),
12 | SC_CREATED(201),
13 | SC_ACCEPTED(202),
14 | SC_NON_AUTHORITATIVE_INFORMATION(203),
15 | SC_NO_CONTENT(204),
16 | SC_RESET_CONTENT(205),
17 | SC_PARTIAL_CONTENT(206),
18 | SC_MULTI_STATUS(207),
19 | SC_MULTIPLE_CHOICES(300),
20 | SC_MOVED_PERMANENTLY(301),
21 | SC_MOVED_TEMPORARILY(302),
22 | SC_SEE_OTHER(303),
23 | SC_NOT_MODIFIED(304),
24 | SC_USE_PROXY(305),
25 | SC_TEMPORARY_REDIRECT(307),
26 | SC_BAD_REQUEST(400),
27 | SC_UNAUTHORIZED(401),
28 | SC_PAYMENT_REQUIRED(402),
29 | SC_FORBIDDEN(403),
30 | SC_NOT_FOUND(404),
31 | SC_METHOD_NOT_ALLOWED(405),
32 | SC_NOT_ACCEPTABLE(406),
33 | SC_PROXY_AUTHENTICATION_REQUIRED(407),
34 | SC_REQUEST_TIMEOUT(408),
35 | SC_CONFLICT(409),
36 | SC_GONE(410),
37 | SC_LENGTH_REQUIRED(411),
38 | SC_PRECONDITION_FAILED(412),
39 | SC_REQUEST_TOO_LONG(413),
40 | SC_REQUEST_URI_TOO_LONG(414),
41 | SC_UNSUPPORTED_MEDIA_TYPE(415),
42 | SC_REQUESTED_RANGE_NOT_SATISFIABLE(416),
43 | SC_EXPECTATION_FAILED(417),
44 | SC_INSUFFICIENT_SPACE_ON_RESOURCE(419),
45 | SC_METHOD_FAILURE(420),
46 | SC_UNPROCESSABLE_ENTITY(422),
47 | SC_LOCKED(423),
48 | SC_FAILED_DEPENDENCY(424),
49 | SC_INTERNAL_SERVER_ERROR(500),
50 | SC_NOT_IMPLEMENTED(501),
51 | SC_BAD_GATEWAY(502),
52 | SC_SERVICE_UNAVAILABLE(503),
53 | SC_GATEWAY_TIMEOUT(504),
54 | SC_HTTP_VERSION_NOT_SUPPORTED(505),
55 | SC_INSUFFICIENT_STORAGE(507);
56 |
57 | private int code;
58 |
59 | HttpStatus(int code) {
60 | this.code = code;
61 | }
62 |
63 | public int getCode() {
64 | return code;
65 | }
66 |
67 | public void setCode(int code) {
68 | this.code = code;
69 | }
70 | }
--------------------------------------------------------------------------------
/src/main/java/com/ruyuapp/util/IpUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.ruyuapp.util;
5 |
6 | import org.slf4j.Logger;
7 | import org.slf4j.LoggerFactory;
8 |
9 | import java.net.*;
10 | import java.util.Enumeration;
11 | import java.util.regex.Pattern;
12 |
13 | /**
14 | * @author letcheng
15 | * @version create at 2016年3月27日 10:47
16 | */
17 | public class IpUtils {
18 |
19 | private static final Logger logger = LoggerFactory.getLogger(IpUtils.class);
20 |
21 | private static class IpUtilsHolder {
22 | private static final InetAddress localAddr = new IpUtils().init();
23 | }
24 |
25 | private IpUtils() {
26 |
27 | }
28 |
29 | public static InetAddress getLocalAddr() {
30 | return IpUtilsHolder.localAddr;
31 | }
32 |
33 | private InetAddress init() {
34 | try {
35 | return InetAddress.getLocalHost(); //直接获取IP地址，适应于Windows机器
36 | } catch (UnknownHostException e) {
37 | e.printStackTrace();
38 | }
39 | Enumeration enumeration = null; //遍历所有的网络接口获取
40 | try {
41 | enumeration = NetworkInterface.getNetworkInterfaces();
42 | } catch (SocketException e) {
43 | e.printStackTrace();
44 | }
45 | while (enumeration != null && enumeration.hasMoreElements()) {
46 | NetworkInterface networkInterface = enumeration.nextElement();
47 | Enumeration addr = networkInterface.getInetAddresses();
48 | while (addr.hasMoreElements()) {
49 | InetAddress localAddr = addr.nextElement();
50 | Pattern IPV4_PATTERN = Pattern.compile("^(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)(\\.(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)){3}$");
51 | if (localAddr.getHostAddress() != null && IPV4_PATTERN.matcher(localAddr.getHostAddress()).matches()) {
52 | return localAddr;
53 | }
54 | }
55 | }
56 | return null;
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/src/main/java/com/ruyuapp/util/ProxyPersistUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.ruyuapp.util;
5 |
6 | import com.ruyuapp.proxy.HttpProxy;
7 | import org.apache.commons.io.FileUtils;
8 | import org.slf4j.Logger;
9 | import org.slf4j.LoggerFactory;
10 |
11 | import java.io.*;
12 | import java.net.InetSocketAddress;
13 | import java.net.Proxy;
14 | import java.util.Map;
15 | import java.util.concurrent.ConcurrentHashMap;
16 |
17 | /**
18 | * 将添加Proxy记录进行持久化保存
19 | *
20 | * @author letcheng
21 | * @version create at 2016年3月27日 23:45
22 | */
23 | public class ProxyPersistUtils {
24 |
25 | private final static Logger logger = LoggerFactory.getLogger(ProxyPersistUtils.class);
26 |
27 | private File path; // 保存proxy的路径
28 |
29 | public ProxyPersistUtils() {
30 | this.path = getFile();
31 | }
32 |
33 | public Map read() {
34 | Map map = new ConcurrentHashMap();
35 | String content = null;
36 | try {
37 | content = FileUtils.readFileToString(this.path);
38 | } catch (IOException e) {
39 | e.printStackTrace();
40 | }
41 | if (content != null && content.length() > 0) {
42 | String lines[] = content.split("\n");
43 | for (String line : lines) {
44 | String tmps[] = line.split(":");
45 | Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(tmps[0], Integer.parseInt(tmps[1])));
46 | HttpProxy httpProxy = new HttpProxy(proxy,
47 | Integer.parseInt(tmps[2]),
48 | Integer.parseInt(tmps[3]),
49 | Integer.parseInt(tmps[4]));
50 | map.put(httpProxy.getKey(), httpProxy);
51 | }
52 | }
53 | return map;
54 | }
55 |
56 | public void save(Map totalQueue) {
57 | if (totalQueue.size() == 0) {
58 | return;
59 | }
60 | StringBuilder stringBuilder = new StringBuilder();
61 | for (Map.Entry entry : totalQueue.entrySet()) {
62 | InetSocketAddress address = (InetSocketAddress) entry.getValue().getProxy().address();
63 | stringBuilder.append(address.getAddress().getHostAddress()).append(":")
64 | .append(address.getPort()).append(":")
65 | .append(entry.getValue().getBorrowNum()).append(":")
66 | .append(entry.getValue().getFailedNum()).append(":")
67 | .append(entry.getValue().getReuseTimeInterval()).append("\n");
68 | }
69 | try {
70 | FileUtils.write(this.path, stringBuilder.toString());
71 | } catch (IOException e) {
72 | e.printStackTrace();
73 | }
74 | }
75 |
76 | private File getFile() {
77 | File file = new File(System.getProperty("java.io.tmpdir") + File.separator + "proxy.tmp");
78 | if (!file.exists()) {
79 | try {
80 | if (!file.createNewFile()) {
81 | throw new IOException(file + "create error");
82 | }
83 | } catch (IOException e) {
84 | e.printStackTrace();
85 | }
86 | }
87 | return file;
88 | }
89 |
90 | }
91 |
--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | log4j.rootCategory=INFO,stdout
2 | log4j.appender.stdout.encoding=utf-8
3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
5 | log4j.appender.stdout.layout.ConversionPattern=[Project] %p [%t] %C.%M(%L) | %m%n
6 | log4j.logger.org.springframework=INFO
7 | log4j.logger.java.sql.ResultSet=INFO
8 | log4j.logger.org.apache=ERROR
9 | log4j.logger.java.sql.Connection=INFO
10 | log4j.logger.java.sql.Statement=INFO
11 | log4j.logger.java.sql.PreparedStatement=INFO
12 |
13 |
--------------------------------------------------------------------------------
/src/test/java/com/ruyuapp/util/IpUtilsTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.ruyuapp.util;
5 |
6 | import com.ruyuapp.proxy.HttpProxy;
7 | import com.ruyuapp.proxy.ProxyPool;
8 | import org.junit.Test;
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 |
12 | import java.net.InetSocketAddress;
13 | import java.net.Proxy;
14 | import java.net.SocketAddress;
15 |
16 | /**
17 | * @author letcheng
18 | * @version create at 2016年3月27日 16:37
19 | */
20 | public class IpUtilsTest {
21 |
22 | private final static Logger logger = LoggerFactory.getLogger(IpUtilsTest.class);
23 |
24 | @Test
25 | public void testGetLocalIp(){
26 | logger.info("Local ip >>> " + IpUtils.getLocalAddr().toString());
27 | }
28 |
29 | }
30 |
--------------------------------------------------------------------------------