├── .gitignore ├── .mvn └── wrapper │ ├── MavenWrapperDownloader.java │ └── maven-wrapper.properties ├── README.md ├── build.sh ├── mvnw ├── mvnw.cmd ├── pom.xml └── src └── main ├── java └── cn │ └── iinti │ └── proxycompose │ ├── Bootstrap.java │ ├── Settings.java │ ├── auth │ ├── AuthRules.java │ └── IpTrie.java │ ├── loop │ ├── Looper.java │ ├── ParallelExecutor.java │ └── ValueCallback.java │ ├── proxy │ ├── ProxyCompose.java │ ├── ProxyServer.java │ ├── RuntimeIpSource.java │ ├── Session.java │ ├── inbound │ │ ├── detector │ │ │ ├── HttpProxyMatcher.java │ │ │ ├── HttpsProxyMatcher.java │ │ │ ├── ProtocolDetector.java │ │ │ ├── ProtocolMatcher.java │ │ │ └── Socks5ProxyMatcher.java │ │ └── handlers │ │ │ ├── ProxyHttpHandler.java │ │ │ ├── ProxySocks5Handler.java │ │ │ └── RelayHandler.java │ ├── outbound │ │ ├── ActiveProxyIp.java │ │ ├── IpPool.java │ │ ├── OutboundOperator.java │ │ ├── downloader │ │ │ ├── DownloadProxyIp.java │ │ │ ├── IpDownloader.java │ │ │ └── IpOfferStep.java │ │ └── handshark │ │ │ ├── AbstractUpstreamHandShaker.java │ │ │ ├── HandShakerHttps.java │ │ │ ├── HandShakerSocks5.java │ │ │ ├── Protocol.java │ │ │ └── ProtocolManager.java │ └── switcher │ │ ├── EatErrorFilter.java │ │ ├── OutboundConnectTask.java │ │ └── UpstreamHandSharkCallback.java │ ├── resource │ ├── DropReason.java │ ├── IpAndPort.java │ ├── IpResourceParser.java │ ├── ProxyIp.java │ └── SmartParser.java │ ├── trace │ ├── Recorder.java │ ├── impl │ │ ├── DiskRecorders.java │ │ ├── SubscribeRecorders.java │ │ └── WheelSlotFilter.java │ └── utils │ │ ├── StringSplitter.java │ │ └── ThrowablePrinter.java │ └── utils │ ├── AsyncHttpInvoker.java │ ├── ConsistentHashUtil.java │ ├── IniConfig.java │ ├── IpUtils.java │ ├── NettyThreadPools.java │ ├── NettyUtil.java │ ├── PortSpaceParser.java │ └── ResourceUtil.java └── resources ├── config.ini └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | node_modules/ 4 | _book/ 5 | *.DS_Store 6 | .logs/ -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | https://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | 20 | import java.io.File; 21 | import java.io.FileInputStream; 22 | import java.io.FileOutputStream; 23 | import java.io.IOException; 24 | import java.net.URL; 25 | import java.nio.channels.Channels; 26 | import java.nio.channels.ReadableByteChannel; 27 | import java.util.Properties; 28 | 29 | public class MavenWrapperDownloader { 30 | 31 | /** 32 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 33 | */ 34 | private static final String DEFAULT_DOWNLOAD_URL = 35 | "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"; 36 | 37 | /** 38 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 39 | * use instead of the default one. 40 | */ 41 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 42 | ".mvn/wrapper/maven-wrapper.properties"; 43 | 44 | /** 45 | * Path where the maven-wrapper.jar will be saved to. 46 | */ 47 | private static final String MAVEN_WRAPPER_JAR_PATH = 48 | ".mvn/wrapper/maven-wrapper.jar"; 49 | 50 | /** 51 | * Name of the property which should be used to override the default download url for the wrapper. 52 | */ 53 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 54 | 55 | public static void main(String args[]) { 56 | System.out.println("- Downloader started"); 57 | File baseDirectory = new File(args[0]); 58 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 59 | 60 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 61 | // wrapperUrl parameter. 62 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 63 | String url = DEFAULT_DOWNLOAD_URL; 64 | if(mavenWrapperPropertyFile.exists()) { 65 | FileInputStream mavenWrapperPropertyFileInputStream = null; 66 | try { 67 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 68 | Properties mavenWrapperProperties = new Properties(); 69 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 70 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 71 | } catch (IOException e) { 72 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 73 | } finally { 74 | try { 75 | if(mavenWrapperPropertyFileInputStream != null) { 76 | mavenWrapperPropertyFileInputStream.close(); 77 | } 78 | } catch (IOException e) { 79 | // Ignore ... 80 | } 81 | } 82 | } 83 | System.out.println("- Downloading from: : " + url); 84 | 85 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 86 | if(!outputFile.getParentFile().exists()) { 87 | if(!outputFile.getParentFile().mkdirs()) { 88 | System.out.println( 89 | "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 90 | } 91 | } 92 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 93 | try { 94 | downloadFileFromURL(url, outputFile); 95 | System.out.println("Done"); 96 | System.exit(0); 97 | } catch (Throwable e) { 98 | System.out.println("- Error downloading"); 99 | e.printStackTrace(); 100 | System.exit(1); 101 | } 102 | } 103 | 104 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 105 | URL website = new URL(urlString); 106 | ReadableByteChannel rbc; 107 | rbc = Channels.newChannel(website.openStream()); 108 | FileOutputStream fos = new FileOutputStream(destination); 109 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 110 | fos.close(); 111 | rbc.close(); 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ProxyCompose 2 | 3 | 这是malenia的开源分支，主要用于给商业代码引流😊 4 | 5 | ProxyCompose是一个对代理IP池进行二次组合的工具，用户可以将多个采购的IP资源进行二次组合，使用统一的账户密码，服务器端口等信息进行统一访问。 6 | 7 | 交流群： 加微信：（iinti_cn）拉入微信交流群 8 | 9 | ## 特性 10 | 11 | 1. 访问统一：如论是什么IP资源供应商，业务代码均接入ProxyCompose，IP资源供应商的变动不会影响业务代码 12 | 2. 池间路由：对于多个IP资源供应商，支持浮动流量比例动态调控，即根据IP池健康评估，弹性伸缩各个IP池的流量。某个IP池挂了不影响整体业务 13 | 3. 协议转换：你可以使用ProxyCompose实现http/https/socks5几种代理协议的转换，这样即使采购的代理资源仅支持socks5，也能转换为https代理 14 | 4. 池化加速：ProxyCompose内置了一个高效的IP池模块，可以对IP资源的访问进行探测、评分、连接池等工作，提高IP资源使用成功率 15 | 16 | ## 使用 17 | 18 | ### 构建 19 | 20 | - 安装Java 21 | - 安装maven 22 | - Linux/mac下，执行脚本：``build.sh``，得到文件``target/proxy-compose.zip``即为产出文件 23 | - 配置： 请根据实际情况配置代理资源 ``conf/config.ini`` 24 | - 运行脚本：``bin/ProxyComposed.sh`` 或 ``bin/ProxyComposed.bat`` 25 | - 代理测试：``curl -x iinti:iinti@127.0.0.1:36000 https://www.baidu.com/`` 26 | 27 | **如不方便构建，可以使用我们构建好的发布包:[https://oss.iinti.cn/malenia/proxy-compose.zip](https://oss.iinti.cn/malenia/proxy-compose.zip)** 28 | 29 | ### 最简配置 30 | 31 | ```ini 32 | [global] 33 | # 鉴权用户，即用户连接到proxy_compose的鉴权 34 | auth_username=iinti 35 | # 鉴权密码 36 | auth_password=iinti 37 | 38 | # 定义IP资源，即从IP供应商采购的IP资源,要求至少配置一个IP资源 39 | [source:dailiyun] 40 | # IP资源下载连接 41 | loadURL=http://修改这里.user.xiecaiyun.com/api/proxies?action=getText&key=修改这里&count=修改这里&word=&rand=false&norepeat=false&detail=false<ime=0 42 | # Ip供应提供的代理账户（如果是白名单，后者无鉴权，则无需配置） 43 | upstreamAuthUser=修改这里 44 | # Ip供应提供的代密码 45 | upstreamAuthPassword=修改这里 46 | # IP池大小，重要 47 | poolSize=10 48 | ``` 49 | 50 | ### 完整配置 51 | 52 | ```ini 53 | [global] 54 | # 开启debug将会有更加丰富的日志 55 | debug=false 56 | # 对代理IP质量进行探测的URL 57 | proxyHttpTestURL = https://iinti.cn/conn/getPublicIp?scene=proxy_compose 58 | # 代理服务器启动端口，本系统将会在配置端口范围连续启动多个代理服务器 59 | mappingSpace=36000-36010 60 | # 是否启用随机隧道，启用随机隧道之后，每次代理请求将会使用随机的IP出口 61 | randomTurning=false 62 | # 是否启用池间路由，池间路由支持根据IP池的健康状态在多个IP池之间动态调整比例 63 | enableFloatIpSourceRatio=true 64 | # failover次数，即系统为失败的代理转发进行的充实 65 | maxFailoverCount=3 66 | # 代理连接超时时间 67 | handleSharkConnectionTimeout=5000 68 | # 鉴权用户，即用户连接到proxy_compose的鉴权 69 | auth_username=iinti 70 | # 鉴权密码 71 | auth_password=iinti 72 | # 使用IP白名单，或者IP端的方式进行鉴权 73 | auth_white_ips=122.23.43.0/24,29.23.45.65 74 | 75 | # 定义IP资源，即从IP供应商采购的IP资源,要求至少配置一个IP资源 76 | # section 要求以 《source:》开始 77 | [source:dailiyun] 78 | # 本资源是否启用，如果希望临时关闭本资源，但是不希望删除配置，可以使用本开关 79 | enable=true 80 | # IP资源下载连接 81 | loadURL=http://修改这里.user.xiecaiyun.com/api/proxies?action=getText&key=修改这里&count=修改这里&word=&rand=false&norepeat=false&detail=false<ime=0 82 | # IP资源格式，目前支持plain，json两种格式，其中json格式需要满足json格式要求 cn.iinti.proxycompose.resource.ProxyIp 83 | resourceFormat=plain 84 | # Ip供应提供的代理账户（如果是白名单，后者无鉴权，则无需配置） 85 | upstreamAuthUser=修改这里 86 | # Ip供应提供的代密码 87 | upstreamAuthPassword=修改这里 88 | # IP池大小，非常重要，此字段为您的IP供应商单次提取返回的节点数 89 | poolSize=10 90 | # 本IP资源池是否需要探测IP质量，如开启，则IP需要被验证可用后方可加入IP池 91 | needTest=true 92 | # IP资源下载间隔时间，单位秒 93 | reloadInterval=240 94 | # IP资源入库后最长存活时间，单位秒，达到此时间后，对应IP资源将会从IP池中移除，除非被重新下载到IP池中 95 | maxAlive=300 96 | # 当前IP资源支持的代理协议（建议至少选择支持socks5的资源） 97 | supportProtocol=socks5,https,http 98 | # 连接池连接空转时间，单位秒，IP池将会提前创建到代理真实代理服务器的连接，给业务使用提供加速功能 99 | connIdleSeconds=20 100 | # 提前创建连接的时间间隔，单位秒 101 | makeConnInterval=20 102 | # 当前IP池在池间流量比例，当存在多个Ip资源配置时，本配置有效，即业务按照此比例对多个IP池进行流量情切 103 | ratio=1 104 | ``` 105 | 106 | ## 特别说明 107 | 108 | ComposeProxy本身能做的工作非常丰富，更多想象空间可以参考我们对应的商业分支：[malenia](https://malenia.iinti.cn/malenia-doc/) 109 | 用户如提交任何新的功能（即使和商业分支重叠）均可以被接收 -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | now_dir=`pwd` 5 | cd `dirname $0` 6 | 7 | shell_dir=`pwd` 8 | 9 | mvn -Dmaven.test.skip=true package appassembler:assemble 10 | 11 | if [[ $? != 0 ]] ;then 12 | echo "build sekiro jar failed" 13 | exit 2 14 | fi 15 | 16 | chmod +x target/proxy-compose/bin/ProxyComposed.sh 17 | proxy_compose_dir=target/proxy-compose 18 | 19 | cd ${proxy_compose_dir} 20 | 21 | zip -r proxy-compose.zip ./* 22 | 23 | mv proxy-compose.zip ../ 24 | 25 | cd ${now_dir} 26 | 27 | echo "the output zip file: target/proxy-compose.zip" -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | ########################################################################################## 204 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 205 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 206 | ########################################################################################## 207 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 208 | if [ "$MVNW_VERBOSE" = true ]; then 209 | echo "Found .mvn/wrapper/maven-wrapper.jar" 210 | fi 211 | else 212 | if [ "$MVNW_VERBOSE" = true ]; then 213 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 214 | fi 215 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" 216 | while IFS="=" read key value; do 217 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 218 | esac 219 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 220 | if [ "$MVNW_VERBOSE" = true ]; then 221 | echo "Downloading from: $jarUrl" 222 | fi 223 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 224 | 225 | if command -v wget > /dev/null; then 226 | if [ "$MVNW_VERBOSE" = true ]; then 227 | echo "Found wget ... using wget" 228 | fi 229 | wget "$jarUrl" -O "$wrapperJarPath" 230 | elif command -v curl > /dev/null; then 231 | if [ "$MVNW_VERBOSE" = true ]; then 232 | echo "Found curl ... using curl" 233 | fi 234 | curl -o "$wrapperJarPath" "$jarUrl" 235 | else 236 | if [ "$MVNW_VERBOSE" = true ]; then 237 | echo "Falling back to using Java to download" 238 | fi 239 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 240 | if [ -e "$javaClass" ]; then 241 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 242 | if [ "$MVNW_VERBOSE" = true ]; then 243 | echo " - Compiling MavenWrapperDownloader.java ..." 244 | fi 245 | # Compiling the Java class 246 | ("$JAVA_HOME/bin/javac" "$javaClass") 247 | fi 248 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 249 | # Running the downloader 250 | if [ "$MVNW_VERBOSE" = true ]; then 251 | echo " - Running MavenWrapperDownloader.java ..." 252 | fi 253 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 254 | fi 255 | fi 256 | fi 257 | fi 258 | ########################################################################################## 259 | # End of extension 260 | ########################################################################################## 261 | 262 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 263 | if [ "$MVNW_VERBOSE" = true ]; then 264 | echo $MAVEN_PROJECTBASEDIR 265 | fi 266 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 267 | 268 | # For Cygwin, switch paths to Windows format before running java 269 | if $cygwin; then 270 | [ -n "$M2_HOME" ] && 271 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 272 | [ -n "$JAVA_HOME" ] && 273 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 274 | [ -n "$CLASSPATH" ] && 275 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 276 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 277 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 278 | fi 279 | 280 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 281 | 282 | exec "$JAVACMD" \ 283 | $MAVEN_OPTS \ 284 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 285 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 286 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 287 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" 124 | FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( 125 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 126 | ) 127 | 128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 130 | if exist %WRAPPER_JAR% ( 131 | echo Found %WRAPPER_JAR% 132 | ) else ( 133 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 134 | echo Downloading from: %DOWNLOAD_URL% 135 | powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" 136 | echo Finished downloading %WRAPPER_JAR% 137 | ) 138 | @REM End of extension 139 | 140 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 141 | if ERRORLEVEL 1 goto error 142 | goto end 143 | 144 | :error 145 | set ERROR_CODE=1 146 | 147 | :end 148 | @endlocal & set ERROR_CODE=%ERROR_CODE% 149 | 150 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 151 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 152 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 153 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 154 | :skipRcPost 155 | 156 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 157 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 158 | 159 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 160 | 161 | exit /B %ERROR_CODE% 162 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 4.0.0 6 | cn.iinti.proxycompose 7 | proxycompose 8 | 0.0.1 9 | jar 10 | 11 | 1.8 12 | 1.8 13 | 31.1-jre 14 | 4.1.60.Final 15 | 16 | 17 | 18 | com.google.code.findbugs 19 | jsr305 20 | 3.0.2 21 | 22 | 23 | org.ini4j 24 | ini4j 25 | 0.5.2 26 | 27 | 28 | ch.qos.logback 29 | logback-classic 30 | 1.2.9 31 | 32 | 33 | com.alibaba 34 | fastjson 35 | 1.2.79 36 | 37 | 38 | org.slf4j 39 | jcl-over-slf4j 40 | 1.7.30 41 | 42 | 43 | org.slf4j 44 | log4j-over-slf4j 45 | 1.7.30 46 | 47 | 48 | org.projectlombok 49 | lombok 50 | 1.18.24 51 | provided 52 | 53 | 54 | com.google.guava 55 | guava 56 | 31.1-jre 57 | 58 | 59 | org.apache.commons 60 | commons-lang3 61 | 3.12.0 62 | 63 | 64 | commons-io 65 | commons-io 66 | 2.10.0 67 | 68 | 69 | 70 | 71 | io.netty 72 | netty-codec-http 73 | ${netty.version} 74 | 75 | 76 | io.netty 77 | netty-tcnative 78 | 1.1.33.Fork26 79 | true 80 | 81 | 82 | io.netty 83 | netty-codec-socks 84 | ${netty.version} 85 | 86 | 87 | 88 | org.asynchttpclient 89 | async-http-client 90 | 2.12.3 91 | 92 | 93 | 94 | ch.hsr 95 | geohash 96 | 1.4.0 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | org.codehaus.mojo 105 | appassembler-maven-plugin 106 | 1.1.1 107 | 108 | flat 109 | lib 110 | src/main/resources 111 | conf 112 | true 113 | true 114 | ${project.build.directory}/proxy-compose 115 | 116 | 117 | -Dfile.encoding=utf-8 -Duser.timezone=GMT+08 -XX:-OmitStackTraceInFastThrow 118 | 119 | 120 | .sh 121 | 122 | 123 | windows 124 | unix 125 | 126 | 127 | 128 | cn.iinti.proxycompose.Bootstrap 129 | ProxyComposed 130 | 131 | 132 | 133 | 134 | 135 | org.apache.maven.plugins 136 | maven-compiler-plugin 137 | 3.8.0 138 | 139 | ${maven.compiler.source} 140 | ${maven.compiler.target} 141 | UTF-8 142 | 1.8 143 | UTF-8 144 | 145 | 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/Bootstrap.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose; 2 | 3 | 4 | import cn.iinti.proxycompose.proxy.ProxyCompose; 5 | import cn.iinti.proxycompose.proxy.RuntimeIpSource; 6 | import cn.iinti.proxycompose.utils.PortSpaceParser; 7 | 8 | import java.io.File; 9 | import java.net.URL; 10 | import java.util.List; 11 | import java.util.TreeSet; 12 | import java.util.stream.Collectors; 13 | 14 | public class Bootstrap { 15 | private static final String IntMessage = 16 | "welcome use ProxyComposed framework, for more support please visit our website: https://iinti.cn/"; 17 | 18 | static { 19 | URL configURL = Bootstrap.class.getClassLoader().getResource("config.ini"); 20 | if (configURL != null && configURL.getProtocol().equals("file")) { 21 | File classPathDir = new File(configURL.getFile()).getParentFile(); 22 | String absolutePath = classPathDir.getAbsolutePath(); 23 | if (absolutePath.endsWith("target/classes") || absolutePath.endsWith("conf")) { 24 | System.setProperty("LOG_DIR", new File(classPathDir.getParentFile(), "logs").getAbsolutePath()); 25 | } 26 | } 27 | } 28 | 29 | public static void main(String[] args) { 30 | List sourceList = Settings.ipSourceList; 31 | if (sourceList.isEmpty()) { 32 | System.err.println("no ipSource defined"); 33 | return; 34 | } 35 | 36 | List runtimeIpSources = sourceList.stream() 37 | .map(RuntimeIpSource::new) 38 | .collect(Collectors.toList()); 39 | 40 | TreeSet ports = PortSpaceParser.parsePortSpace(Settings.global.mappingSpace.value); 41 | if (ports.isEmpty()) { 42 | System.err.println("proxy server port not config"); 43 | return; 44 | } 45 | new ProxyCompose(runtimeIpSources, ports); 46 | System.out.println(IntMessage); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/Settings.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose; 2 | 3 | import cn.iinti.proxycompose.auth.AuthRules; 4 | import cn.iinti.proxycompose.utils.IniConfig; 5 | import cn.iinti.proxycompose.utils.IpUtils; 6 | import cn.iinti.proxycompose.utils.ResourceUtil; 7 | import lombok.Getter; 8 | import lombok.SneakyThrows; 9 | import org.apache.commons.lang3.StringUtils; 10 | import org.ini4j.ConfigParser; 11 | 12 | import java.io.InputStream; 13 | import java.net.SocketException; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | 17 | /** 18 | * 配置文件解析和配置内容定义 19 | */ 20 | public class Settings { 21 | 22 | /** 23 | * 全局配置 24 | */ 25 | public static Global global; 26 | /** 27 | * IP资源定义，系统至少包含1个IP资源 28 | */ 29 | public static List ipSourceList = new ArrayList<>(); 30 | 31 | static { 32 | load(); 33 | } 34 | 35 | @SneakyThrows 36 | private static void load() { 37 | InputStream stream = ResourceUtil.openResource("config.ini"); 38 | 39 | ConfigParser config = new ConfigParser(); 40 | config.read(stream); 41 | 42 | global = new Global(config); 43 | 44 | List sections = config.sections(); 45 | for (String sectionName : sections) { 46 | if (sectionName.startsWith("source:")) { 47 | ipSourceList.add(new IpSource(config, sectionName)); 48 | } 49 | } 50 | } 51 | 52 | @Getter 53 | public static class IpSource extends IniConfig { 54 | 55 | public IpSource(ConfigParser config, String section) { 56 | super(config, section); 57 | name = section.substring("source:".length()); 58 | } 59 | 60 | public final String name; 61 | 62 | public final StringConfigValue loadURL = new StringConfigValue( 63 | "loadURL", ""); 64 | 65 | /** 66 | * 资源格式，目前支持两种 67 | * plain：文本分割，如 proxy.iinti.cn:8000-9000,proxy1.iinti.cn:5817,proxy2.iinti.cn:5817 68 | * json: json格式，满足 {@link cn.iinti.proxycompose.resource.ProxyIp}的json对象 69 | */ 70 | public final StringConfigValue resourceFormat = new StringConfigValue( 71 | "resourceFormat", "plain" 72 | ); 73 | 74 | public final BooleanConfigValue enable = new BooleanConfigValue( 75 | "enable", false 76 | ); 77 | 78 | public final StringConfigValue upstreamAuthUser = new StringConfigValue( 79 | "upstreamAuthUser", ""); 80 | 81 | public final StringConfigValue upstreamAuthPassword = new StringConfigValue( 82 | "upstreamAuthPassword", ""); 83 | 84 | public final IntegerConfigValue poolSize = new IntegerConfigValue( 85 | "poolSize", 0 86 | ); 87 | 88 | public final BooleanConfigValue needTest = new BooleanConfigValue( 89 | "needTest", true 90 | ); 91 | public final IntegerConfigValue reloadInterval = new IntegerConfigValue( 92 | "reloadInterval", 60); 93 | 94 | public IntegerConfigValue maxAlive = new IntegerConfigValue( 95 | "maxAlive", 1200 96 | ); 97 | 98 | public final StringConfigValue supportProtocol = new StringConfigValue( 99 | "supportProtocol", "socks5"); 100 | 101 | public final IntegerConfigValue connIdleSeconds = new IntegerConfigValue( 102 | "connIdleSeconds", 5); 103 | 104 | public final IntegerConfigValue makeConnInterval = new IntegerConfigValue( 105 | "makeConnInterval", 500); 106 | 107 | public final IntegerConfigValue ratio = new IntegerConfigValue( 108 | "ratio", 1 109 | ); 110 | 111 | } 112 | 113 | public static class Global extends IniConfig { 114 | 115 | public BooleanConfigValue debug = new BooleanConfigValue( 116 | "debug", false 117 | ); 118 | 119 | /** 120 | * 后端探测接口，探测代理ip是否可用以及解析出口ip地址 121 | */ 122 | public StringConfigValue proxyHttpTestURL = new StringConfigValue( 123 | "proxyHttpTestURL", "https://iinti.cn/conn/getPublicIp?scene=proxy_compose"); 124 | 125 | 126 | public StringConfigValue mappingSpace = new StringConfigValue( 127 | "mappingSpace", "36000-36010" 128 | ); 129 | 130 | public BooleanConfigValue randomTurning = new BooleanConfigValue( 131 | "randomTurning", false 132 | ); 133 | 134 | public BooleanConfigValue enableFloatIpSourceRatio = new BooleanConfigValue( 135 | "enableFloatIpSourceRatio", true 136 | ); 137 | 138 | public IntegerConfigValue maxFailoverCount = new IntegerConfigValue( 139 | "maxFailoverCount", 3 140 | ); 141 | 142 | public IntegerConfigValue handleSharkConnectionTimeout = new IntegerConfigValue( 143 | "handleSharkConnectionTimeout", 5_000 144 | ); 145 | 146 | public final AuthRules authRules = new AuthRules(); 147 | 148 | public String listenIp = "0.0.0.0"; 149 | 150 | public Global(ConfigParser config) { 151 | super(config, "global"); 152 | acceptConfig("auth_username", authRules::setAuthAccount); 153 | acceptConfig("auth_password", authRules::setAuthPassword); 154 | acceptConfig("auth_white_ips", s -> { 155 | for (String str : StringUtils.split(s, ",")) { 156 | if (StringUtils.isBlank(str)) { 157 | continue; 158 | } 159 | authRules.addCidrIpConfig(str.trim()); 160 | } 161 | }); 162 | acceptConfig("listen_type", listenType -> listenIp = parseListenType(listenType)); 163 | } 164 | } 165 | 166 | private static String parseListenType(String listenType) { 167 | listenType = StringUtils.trimToEmpty(listenType); 168 | if (IpUtils.isIpV4(listenType)) { 169 | return listenType; 170 | } 171 | switch (listenType) { 172 | //lo,private,public,all 173 | case "lo": 174 | return "127.0.0.1"; 175 | 176 | case "all": 177 | return "0.0.0.0"; 178 | default: 179 | try { 180 | return IpUtils.fetchIp(listenType); 181 | } catch (SocketException e) { 182 | //ignore 183 | } 184 | } 185 | return "0.0.0.0"; 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/auth/AuthRules.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.auth; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | public class AuthRules { 8 | /** 9 | * IP匹配前缀树 10 | */ 11 | private final IpTrie ipTrie = new IpTrie(); 12 | 13 | 14 | @Getter 15 | @Setter 16 | private String authAccount; 17 | 18 | @Getter 19 | @Setter 20 | private String authPassword; 21 | 22 | public void addCidrIpConfig(String ipConfig) { 23 | ipTrie.insert(ipConfig); 24 | } 25 | 26 | 27 | public boolean doAuth(String ip) { 28 | return ipTrie.has(ip); 29 | } 30 | 31 | public boolean doAuth(String authUserName, String authPassword) { 32 | return StringUtils.equals(this.authAccount, authUserName) 33 | && StringUtils.equals(this.authPassword, authPassword); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/auth/IpTrie.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.auth; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | public class IpTrie { 6 | private IpTrie left; 7 | private IpTrie right; 8 | private boolean has = false; 9 | private static final String localhostStr = "localhost"; 10 | private static final String localhost = "127.0.0.1"; 11 | 12 | 13 | public void insert(String ipConfig) { 14 | String ip; 15 | int cidr = 32; 16 | if (ipConfig.contains("/")) { 17 | String[] split = ipConfig.split("/"); 18 | ip = StringUtils.trim(split[0]); 19 | cidr = Integer.parseInt(StringUtils.trim(split[1])); 20 | } else { 21 | ip = ipConfig.trim(); 22 | } 23 | 24 | insert(0, ip2Int(ip), cidr); 25 | } 26 | 27 | private void insert(int deep, long ip, int cidr) { 28 | if (deep >= cidr) { 29 | has = true; 30 | return; 31 | } 32 | 33 | int bit = (int) ((ip >>> (32 - deep)) & 0x01); 34 | if (bit == 0) { 35 | if (left == null) { 36 | left = new IpTrie(); 37 | } 38 | left.insert(deep + 1, ip, cidr); 39 | } else { 40 | if (right == null) { 41 | right = new IpTrie(); 42 | } 43 | right.insert(deep + 1, ip, cidr); 44 | } 45 | } 46 | 47 | private static long ip2Int(String ip) { 48 | if (localhostStr.equals(ip)) { 49 | ip = localhost; 50 | } 51 | String[] split = ip.split("\\."); 52 | return ((Long.parseLong(split[0]) << 24 53 | | Long.parseLong(split[1]) << 16 54 | | Long.parseLong(split[2]) << 8 55 | | Long.parseLong(split[3]))); 56 | } 57 | 58 | 59 | public boolean has(String ip) { 60 | if (ip.contains("/")) { 61 | // 这里可能输入了一个cidr的ip规则 62 | ip = ip.substring(0, ip.indexOf('/')); 63 | } 64 | 65 | return has(ip2Int(ip), 0); 66 | } 67 | 68 | 69 | private boolean has(long ip, int deep) { 70 | if (has) { 71 | return true; 72 | } 73 | int bit = (int) ((ip >>> (32 - deep)) & 0x01); 74 | if (bit == 0) { 75 | if (left == null) { 76 | return false; 77 | } 78 | return left.has(ip, deep + 1); 79 | } else { 80 | if (right == null) { 81 | return false; 82 | } 83 | return right.has(ip, deep + 1); 84 | } 85 | } 86 | 87 | public void remove(String ipConfig) { 88 | String ip; 89 | int cidr = 32; 90 | if (ipConfig.contains("/")) { 91 | String[] split = ipConfig.split("/"); 92 | cidr = Integer.parseInt(StringUtils.trim(split[1])); 93 | ip = StringUtils.trim(split[0]); 94 | } else { 95 | ip = ipConfig.trim(); 96 | } 97 | remove(0, ip2Int(ip), cidr); 98 | } 99 | 100 | private void remove(int deep, long ip, int cidr) { 101 | if (deep >= cidr) { 102 | has = false; 103 | return; 104 | } 105 | 106 | int bit = (int) ((ip >>> (32 - deep)) & 0x01); 107 | if (bit == 0) { 108 | if (left == null) { 109 | return; 110 | } 111 | left.remove(deep + 1, ip, cidr); 112 | } else { 113 | if (right == null) { 114 | return; 115 | } 116 | right.remove(deep + 1, ip, cidr); 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/loop/Looper.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.loop; 2 | 3 | 4 | import lombok.extern.slf4j.Slf4j; 5 | 6 | import javax.annotation.Nonnull; 7 | import java.util.concurrent.*; 8 | 9 | /** 10 | * 单线程事件循环模型，用来避免一致性问题 11 | */ 12 | @Slf4j 13 | public class Looper implements Executor { 14 | 15 | private final LinkedBlockingDeque taskQueue = new LinkedBlockingDeque<>(); 16 | 17 | private final LoopThread loopThread; 18 | private final long createTimestamp = System.currentTimeMillis(); 19 | 20 | private static Looper lowPriorityLooper; 21 | 22 | 23 | /** 24 | * 获取一个全局的低优looper 25 | * 26 | * @return looper 27 | */ 28 | public static Looper getLowPriorityLooper() { 29 | if (lowPriorityLooper != null) { 30 | return lowPriorityLooper; 31 | } 32 | synchronized (Looper.class) { 33 | lowPriorityLooper = new Looper("lowPriorityLooper").startLoop(); 34 | } 35 | return lowPriorityLooper; 36 | } 37 | 38 | /** 39 | * 让looper拥有延时任务的能力 40 | */ 41 | private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); 42 | 43 | public Looper(String looperName) { 44 | loopThread = new LoopThread(looperName); 45 | loopThread.setDaemon(true); 46 | } 47 | 48 | 49 | public Looper startLoop() { 50 | loopThread.start(); 51 | return this; 52 | } 53 | 54 | public void post(Runnable runnable) { 55 | post(runnable, false); 56 | } 57 | 58 | public void offerLast(Runnable runnable) { 59 | taskQueue.addLast(runnable); 60 | } 61 | 62 | public void post(Runnable runnable, boolean first) { 63 | if (!loopThread.isAlive()) { 64 | if (System.currentTimeMillis() - createTimestamp > 60000) { 65 | log.warn("post task before looper startup,do you call :startLoop??", new Throwable()); 66 | } 67 | runnable.run(); 68 | return; 69 | } 70 | if (inLooper()) { 71 | runnable.run(); 72 | return; 73 | } 74 | if (first) { 75 | taskQueue.offerFirst(runnable); 76 | } else { 77 | taskQueue.add(runnable); 78 | } 79 | } 80 | 81 | public void postDelay(Runnable runnable, long delay) { 82 | if (delay <= 0) { 83 | post(runnable); 84 | return; 85 | } 86 | if (!loopThread.isAlive()) { 87 | //todo 这是应该是有bug，先加上这一行日志 88 | if (System.currentTimeMillis() - createTimestamp > 60000) { 89 | log.warn("post task before looper startup,do you call :startLoop??", new Throwable()); 90 | } 91 | } 92 | scheduler.schedule(() -> post(runnable), delay, TimeUnit.MILLISECONDS); 93 | } 94 | 95 | public Looper fluentScheduleWithRate(Runnable runnable, long rate) { 96 | scheduleWithRate(runnable, rate); 97 | return this; 98 | } 99 | 100 | public FixRateScheduleHandle scheduleWithRate(Runnable runnable, long rate) { 101 | return scheduleWithRate(runnable, Long.valueOf(rate)); 102 | } 103 | 104 | /** 105 | * 这个接口，可以支持非固定速率 106 | */ 107 | public FixRateScheduleHandle scheduleWithRate(Runnable runnable, Number rate) { 108 | FixRateScheduleHandle fixRateScheduleHandle = new FixRateScheduleHandle(runnable, rate); 109 | // if (rate.longValue() > 0) { 110 | // post(runnable); 111 | // } 112 | postDelay(fixRateScheduleHandle, rate.longValue()); 113 | return fixRateScheduleHandle; 114 | } 115 | 116 | @Override 117 | public void execute(@Nonnull Runnable command) { 118 | if (inLooper()) { 119 | command.run(); 120 | return; 121 | } 122 | post(command); 123 | } 124 | 125 | 126 | public class FixRateScheduleHandle implements Runnable { 127 | private final Runnable runnable; 128 | private final Number rate; 129 | private boolean running; 130 | 131 | 132 | FixRateScheduleHandle(Runnable runnable, Number rate) { 133 | this.runnable = runnable; 134 | this.running = true; 135 | this.rate = rate; 136 | } 137 | 138 | public void cancel() { 139 | this.running = false; 140 | } 141 | 142 | @Override 143 | public void run() { 144 | if (running && rate.longValue() > 0) { 145 | postDelay(this, rate.longValue()); 146 | } 147 | runnable.run(); 148 | } 149 | 150 | } 151 | 152 | public boolean inLooper() { 153 | return Thread.currentThread().equals(loopThread) 154 | || !loopThread.isAlive(); 155 | } 156 | 157 | public void checkLooper() { 158 | if (!inLooper()) { 159 | throw new IllegalStateException("run task not in looper"); 160 | } 161 | } 162 | 163 | private class LoopThread extends Thread { 164 | LoopThread(String name) { 165 | super(name); 166 | } 167 | 168 | private boolean run = true; 169 | 170 | @Override 171 | public void run() { 172 | while (run) { 173 | try { 174 | taskQueue.take().run(); 175 | } catch (InterruptedException interruptedException) { 176 | return; 177 | } catch (Throwable throwable) { 178 | log.error("group event loop error", throwable); 179 | } 180 | } 181 | } 182 | } 183 | 184 | public void close() { 185 | loopThread.run = false; 186 | loopThread.interrupt(); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/loop/ParallelExecutor.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.loop; 2 | 3 | public class ParallelExecutor implements ValueCallback { 4 | private final Looper looper; 5 | 6 | private final int eventSize; 7 | private int eventIndex = 0; 8 | private boolean success = false; 9 | private final ParallelConnectEvent parallelConnectEvent; 10 | 11 | public ParallelExecutor(Looper looper, int eventSize, ParallelConnectEvent parallelConnectEvent) { 12 | this.looper = looper; 13 | this.eventSize = eventSize; 14 | this.parallelConnectEvent = parallelConnectEvent; 15 | } 16 | 17 | @Override 18 | public void onReceiveValue(Value value) { 19 | looper.execute(() -> { 20 | eventIndex++; 21 | 22 | if (value.isSuccess()) { 23 | if (!success) { 24 | success = true; 25 | parallelConnectEvent.firstSuccess(value); 26 | } else { 27 | parallelConnectEvent.secondSuccess(value); 28 | } 29 | return; 30 | } 31 | 32 | if (!success && eventIndex >= eventSize) { 33 | parallelConnectEvent.finalFailed(value.e); 34 | } 35 | }); 36 | 37 | 38 | } 39 | 40 | //private List<> 41 | 42 | public interface ParallelConnectEvent { 43 | void firstSuccess(Value value); 44 | 45 | void secondSuccess(Value value); 46 | 47 | void finalFailed(Throwable throwable); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/loop/ValueCallback.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.loop; 2 | 3 | 4 | public interface ValueCallback { 5 | void onReceiveValue(Value value); 6 | 7 | 8 | static void success(ValueCallback callback, T t) { 9 | callback.onReceiveValue(Value.success(t)); 10 | } 11 | 12 | static void failed(ValueCallback callback, Throwable e) { 13 | callback.onReceiveValue(Value.failed(e)); 14 | } 15 | 16 | static void failed(ValueCallback callback, String message) { 17 | callback.onReceiveValue(Value.failed(message)); 18 | } 19 | 20 | class Value { 21 | 22 | public T v; 23 | public Throwable e; 24 | 25 | public boolean isSuccess() { 26 | return e == null; 27 | } 28 | 29 | public static Value failed(Throwable e) { 30 | Value value = new Value<>(); 31 | value.e = e; 32 | return value; 33 | } 34 | 35 | public static Value failed(String message) { 36 | return failed(new RuntimeException(message)); 37 | } 38 | 39 | public static Value success(T t) { 40 | Value value = new Value<>(); 41 | value.v = t; 42 | return value; 43 | } 44 | 45 | public Value errorTransfer() { 46 | Value value = new Value<>(); 47 | value.e = e; 48 | return value; 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/ProxyCompose.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy; 2 | 3 | import cn.iinti.proxycompose.Settings; 4 | import cn.iinti.proxycompose.proxy.outbound.ActiveProxyIp; 5 | import cn.iinti.proxycompose.utils.ConsistentHashUtil; 6 | import cn.iinti.proxycompose.loop.Looper; 7 | import cn.iinti.proxycompose.loop.ValueCallback; 8 | import cn.iinti.proxycompose.trace.Recorder; 9 | import cn.iinti.proxycompose.trace.impl.SubscribeRecorders; 10 | import com.alibaba.fastjson.JSONObject; 11 | import com.google.common.cache.Cache; 12 | import com.google.common.cache.CacheBuilder; 13 | import com.google.common.cache.RemovalListener; 14 | import com.google.common.collect.Lists; 15 | import com.google.common.collect.Maps; 16 | import io.netty.channel.Channel; 17 | import lombok.Getter; 18 | import lombok.extern.slf4j.Slf4j; 19 | 20 | import java.util.*; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | @Slf4j 24 | public class ProxyCompose { 25 | /** 26 | * 所有的代理IP池，IP池代表malenia的上有资源池，他来自各种代理ip供应商。如代理云、芝麻代理、快代理等 27 | * malenia通过对IpSource的抽象，完成对所有代理供应商的透明屏蔽 28 | */ 29 | private final Map ipSources = Maps.newHashMap(); 30 | 31 | /** 32 | * 本产品下的所有代理服务器 33 | */ 34 | private final TreeMap proxyServerTreeMap = Maps.newTreeMap(); 35 | 36 | private TreeMap ipSourceKeyMap = new TreeMap<>(); 37 | private List ipSourceKeyList = Lists.newArrayList(); 38 | /** 39 | * 路由缓存，当一个隧道访问过程使用过某个ip资源，那么系统优先尝试使用曾今的隧道，如此尽可能保证ip出口不变
40 | * ps：需要做这个缓存更重要的原因是，malenia在运作过程会有failover，failover过程会有不可预期的隧道映射关系重置 41 | * 此时无法根据固定的规则进行mapping计算 42 | */ 43 | private final Cache routeCache = CacheBuilder.newBuilder() 44 | .removalListener((RemovalListener) notification -> { 45 | Long sessionHash = notification.getKey(); 46 | ActiveProxyIp activeProxyIp = notification.getValue(); 47 | // auto decrease refCount of proxyIp 48 | if (sessionHash != null && activeProxyIp != null) { 49 | activeProxyIp.refreshRefSessionHash(sessionHash, false); 50 | } 51 | }) 52 | .expireAfterAccess(1, TimeUnit.MINUTES).build(); 53 | 54 | 55 | @Getter 56 | private final Looper composeWorkThead; 57 | 58 | public ProxyCompose(List ipSources, Set proxyServePorts) { 59 | composeWorkThead = new Looper("compose").startLoop(); 60 | for (Integer port : proxyServePorts) { 61 | ProxyServer proxyServer = new ProxyServer(port, ProxyCompose.this); 62 | log.info("start proxy server for port: {}", port); 63 | proxyServerTreeMap.put(port, proxyServer); 64 | } 65 | 66 | // config ipSource 67 | Map ipSourceWithRatio = Maps.newHashMap(); 68 | for (RuntimeIpSource runtimeIpSource : ipSources) { 69 | String sourceKey = runtimeIpSource.getName(); 70 | this.ipSources.put(sourceKey, runtimeIpSource); 71 | ipSourceWithRatio.put(sourceKey, runtimeIpSource.getRatio().value); 72 | } 73 | Map ratioConfig = Collections.unmodifiableMap(ipSourceWithRatio); 74 | reloadIpSourceRatio(ratioConfig); 75 | composeWorkThead.scheduleWithRate(() -> reloadIpSourceRatio(ratioConfig), 300_000); 76 | } 77 | 78 | 79 | private Map floatRatio(Map configRule) { 80 | configRule = new HashMap<>(configRule); 81 | 82 | boolean hasVerySmallConfig = configRule.values().stream().anyMatch(it -> it.equals(1) || it.equals(2)); 83 | if (hasVerySmallConfig) { 84 | // expand ratio, because the float ratio component maybe decease config 85 | for (String ipSourceKey : Lists.newArrayList(configRule.keySet())) { 86 | configRule.put(ipSourceKey, configRule.get(ipSourceKey) * 3); 87 | } 88 | } 89 | 90 | // scale ratio config by health score 91 | for (String ipSourceKey : Lists.newArrayList(configRule.keySet())) { 92 | //IP池健康指数，正常值为100，最大值一般不超过150，小于100认为不健康，最低为0（代表此IP资源已经完全挂了） 93 | RuntimeIpSource runtimeIpSource = ipSources.get(ipSourceKey); 94 | if (runtimeIpSource == null) { 95 | continue; 96 | } 97 | Double healthScore = runtimeIpSource.healthScore(); 98 | 99 | Integer configuredRatio = configRule.get(ipSourceKey); 100 | configuredRatio = (int) (configuredRatio * healthScore / 100); 101 | if (configuredRatio <= 1) { 102 | configuredRatio = 1; 103 | } 104 | configRule.put(ipSourceKey, configuredRatio); 105 | } 106 | return configRule; 107 | } 108 | 109 | public void reloadIpSourceRatio(Map ratio) { 110 | composeWorkThead.post(() -> { 111 | Map configRule = Settings.global.enableFloatIpSourceRatio.value ? 112 | floatRatio(ratio) : ratio; 113 | 114 | TreeMap newIpSources = new TreeMap<>(); 115 | List newIpSourceLists = Lists.newArrayList(new TreeSet<>(configRule.keySet())); 116 | 117 | for (String ipSourceKey : configRule.keySet()) { 118 | int ratio1 = configRule.get(ipSourceKey); 119 | if (ratio1 == 0) { 120 | continue; 121 | } 122 | for (int i = 1; i <= ratio1; i++) { 123 | long murHash = ConsistentHashUtil.murHash(ipSourceKey + "_##_" + i); 124 | newIpSources.put(murHash, ipSourceKey); 125 | } 126 | } 127 | ipSourceKeyMap = newIpSources; 128 | ipSourceKeyList = newIpSourceLists; 129 | }); 130 | } 131 | 132 | 133 | public void destroy() { 134 | composeWorkThead.post(() -> { 135 | proxyServerTreeMap.values().forEach(ProxyServer::destroy); 136 | composeWorkThead.close(); 137 | }); 138 | } 139 | 140 | public void connectToOutbound( 141 | Session session, long sessionHash, String tag, 142 | ActiveProxyIp.ActivityProxyIpBindObserver observer, 143 | ValueCallback callback) { 144 | SubscribeRecorders.SubscribeRecorder recorder = session.getRecorder(); 145 | 146 | allocateIpSource(sessionHash, tag, session, ipSourceValue -> { 147 | if (!ipSourceValue.isSuccess()) { 148 | recorder.recordEvent(() -> tag + "allocate IpSource failed", ipSourceValue.e); 149 | ValueCallback.failed(callback, ipSourceValue.e); 150 | return; 151 | } 152 | 153 | RuntimeIpSource ipSource = ipSourceValue.v; 154 | // 拿到IP源，此时ip源是根据分流比例控制的 155 | recorder.recordMosaicMsg(() -> tag + "allocate IpSource success: " + ipSource.getName()); 156 | // 在ip源上分配代理 157 | ipSource.getIpPool().allocateIp(sessionHash, session.getRecorder(), ipWrapper -> { 158 | if (!ipWrapper.isSuccess()) { 159 | recorder.recordEvent(() -> tag + "allocate IP failed"); 160 | ValueCallback.failed(callback, ipWrapper.e); 161 | return; 162 | } 163 | 164 | ActiveProxyIp activeProxyIp = ipWrapper.v; 165 | recorder.recordMosaicMsg(() -> tag + "allocate IP success:" + 166 | JSONObject.toJSONString(activeProxyIp.getDownloadProxyIp()) 167 | + " begin to borrow connection" 168 | ); 169 | activeProxyIp.borrowConnect(recorder, tag, observer, callback); 170 | }); 171 | }); 172 | } 173 | 174 | public void allocateIpSource(long sessionHash, String tag, Session session, ValueCallback valueCallback) { 175 | composeWorkThead.post(() -> { 176 | // 如果用户有指定了代理ip源，那么使用特定的代理ip 177 | String ipSourceKey = ConsistentHashUtil.fetchConsistentRing(ipSourceKeyMap, sessionHash); 178 | if (ipSourceKey == null) { 179 | // not happen 180 | valueCallback.onReceiveValue(ValueCallback.Value.failed("no ipSources mapping for ")); 181 | return; 182 | } 183 | session.getRecorder().recordMosaicMsg(() -> tag + "map route to ipSource: " + ipSourceKey); 184 | allocateIpSource0(session.getRecorder(), tag, ipSourceKey, ipSourceKeyList, valueCallback); 185 | }); 186 | } 187 | 188 | private void allocateIpSource0(Recorder recorder, String tag, String prefer, List candidate, 189 | ValueCallback valueCallback) { 190 | RuntimeIpSource runtimeIpSource = ipSources.get(prefer); 191 | 192 | if (runtimeIpSource == null) { 193 | valueCallback.onReceiveValue(ValueCallback.Value.failed("no ipSources: " + prefer + " defined")); 194 | return; 195 | } 196 | if (isValidIpSource(runtimeIpSource)) { 197 | ValueCallback.success(valueCallback, runtimeIpSource); 198 | return; 199 | } 200 | if (ipSources.size() > 1) { 201 | recorder.recordEvent(() -> tag + "IpSourceManager this ipSource not have ip resource, begin ratio to next ip source"); 202 | int length = candidate.size(); 203 | int start = Math.abs(prefer.hashCode()) + candidate.size(); 204 | for (int i = 0; i < length; i++) { 205 | String newSourceKey = candidate.get((start + i) % candidate.size()); 206 | if (newSourceKey.equals(prefer)) { 207 | continue; 208 | } 209 | runtimeIpSource = ipSources.get(newSourceKey); 210 | if (isValidIpSource(runtimeIpSource)) { 211 | ValueCallback.success(valueCallback, runtimeIpSource); 212 | return; 213 | } 214 | } 215 | } 216 | 217 | ValueCallback.failed(valueCallback, "can not find online IpSourceKey"); 218 | } 219 | 220 | public static boolean isValidIpSource(RuntimeIpSource runtimeIpSource) { 221 | return runtimeIpSource != null && !runtimeIpSource.poolEmpty(); 222 | } 223 | 224 | public void fetchCachedSession(Session session, ValueCallback callback) { 225 | composeWorkThead.post(() -> { 226 | ActiveProxyIp sessionIp = routeCache.getIfPresent(session.getSessionHash()); 227 | if (sessionIp == null) { 228 | ValueCallback.failed(callback, "not exist"); 229 | return; 230 | } 231 | 232 | if (sessionIp.getActiveStatus() != ActiveProxyIp.ActiveStatus.DESTROY) { 233 | ValueCallback.success(callback, sessionIp); 234 | return; 235 | } 236 | routeCache.invalidate(session.getSessionHash()); 237 | ValueCallback.failed(callback, "not exist"); 238 | }); 239 | 240 | } 241 | 242 | public void markSessionUse(Session session, ActiveProxyIp activeProxyIp) { 243 | if (Settings.global.randomTurning.value) { 244 | return; 245 | } 246 | session.getRecorder().recordEvent(() -> "add sessionId route mapping "); 247 | activeProxyIp.refreshRefSessionHash(session.getSessionHash(), true); 248 | composeWorkThead.post(() -> routeCache.put(session.getSessionHash(), activeProxyIp)); 249 | } 250 | } -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/ProxyServer.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy; 2 | 3 | import cn.iinti.proxycompose.proxy.inbound.detector.HttpProxyMatcher; 4 | import cn.iinti.proxycompose.proxy.inbound.detector.HttpsProxyMatcher; 5 | import cn.iinti.proxycompose.proxy.inbound.detector.ProtocolDetector; 6 | import cn.iinti.proxycompose.proxy.inbound.detector.Socks5ProxyMatcher; 7 | import cn.iinti.proxycompose.proxy.inbound.handlers.ProxyHttpHandler; 8 | import cn.iinti.proxycompose.proxy.inbound.handlers.ProxySocks5Handler; 9 | import cn.iinti.proxycompose.utils.NettyThreadPools; 10 | import cn.iinti.proxycompose.utils.NettyUtil; 11 | import cn.iinti.proxycompose.trace.Recorder; 12 | import io.netty.bootstrap.ServerBootstrap; 13 | import io.netty.buffer.Unpooled; 14 | import io.netty.channel.*; 15 | import io.netty.channel.socket.SocketChannel; 16 | import io.netty.channel.socket.nio.NioServerSocketChannel; 17 | import lombok.Getter; 18 | import lombok.extern.slf4j.Slf4j; 19 | 20 | import java.lang.ref.WeakReference; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | 24 | 25 | /** 26 | * 描述一个代理服务器 27 | */ 28 | @Slf4j 29 | public class ProxyServer { 30 | public static final byte[] UN_SUPPORT_PROTOCOL_MSG = "unknown protocol".getBytes(); 31 | 32 | @Getter 33 | public int port; 34 | 35 | 36 | private Channel serverChannel; 37 | 38 | @Getter 39 | private final ProxyCompose proxyCompose; 40 | 41 | public ProxyServer(int port, ProxyCompose proxyCompose) { 42 | this.port = port; 43 | this.proxyCompose = proxyCompose; 44 | startProxy(buildProxyServerConfig(), 20); 45 | } 46 | 47 | private void startProxy(ServerBootstrap serverBootstrap, int leftRetry) { 48 | if (leftRetry < 0) { 49 | log.error("the proxy server start failed finally!!:{}", port); 50 | return; 51 | } 52 | serverBootstrap.bind(port).addListener((ChannelFutureListener) future -> { 53 | if (future.isSuccess()) { 54 | log.info("proxy server start success: {}", port); 55 | serverChannel = future.channel(); 56 | return; 57 | } 58 | log.info("proxy server start failed, will be retry after 5s", future.cause()); 59 | future.channel().eventLoop().schedule(() -> startProxy( 60 | serverBootstrap, leftRetry - 1), 5, TimeUnit.SECONDS 61 | ); 62 | }); 63 | } 64 | 65 | public boolean enable() { 66 | return serverChannel != null && serverChannel.isActive(); 67 | } 68 | 69 | 70 | private ServerBootstrap buildProxyServerConfig() { 71 | return new ServerBootstrap() 72 | .group(NettyThreadPools.proxyServerBossGroup, NettyThreadPools.proxyServerWorkerGroup) 73 | .channel(NioServerSocketChannel.class) 74 | .childHandler(new ChannelInitializer() { 75 | @Override 76 | protected void initChannel(SocketChannel ch) { 77 | Session session = Session.touch(ch, ProxyServer.this); 78 | ch.pipeline().addLast( 79 | buildProtocolDetector(session.getRecorder()) 80 | ); 81 | setupProtocolTimeoutCheck(ch, session.getRecorder()); 82 | } 83 | }); 84 | } 85 | 86 | private static void setupProtocolTimeoutCheck(Channel channel, Recorder recorder) { 87 | WeakReference ref = new WeakReference<>(channel); 88 | EventLoop eventLoop = channel.eventLoop(); 89 | 90 | eventLoop.schedule(() -> 91 | doHandlerTimeoutCheck(ref, ProtocolDetector.class, recorder, "protocol detect timeout, close this channel"), 92 | 5, TimeUnit.SECONDS); 93 | 94 | eventLoop.schedule(() -> { 95 | doHandlerTimeoutCheck(ref, ProxyHttpHandler.class, recorder, "http proxy init timeout"); 96 | doHandlerTimeoutCheck(ref, ProxySocks5Handler.class, recorder, "socks5 proxy init timeout"); 97 | }, 2, TimeUnit.MINUTES); 98 | 99 | } 100 | 101 | private static void doHandlerTimeoutCheck( 102 | WeakReference ref, Class handlerClazz, Recorder recorder, String msgHit 103 | ) { 104 | Channel ch = ref.get(); 105 | if (ch == null) { 106 | return; 107 | } 108 | T handler = ch.pipeline().get(handlerClazz); 109 | if (handler != null) { 110 | recorder.recordEvent(msgHit); 111 | ch.close(); 112 | } 113 | } 114 | 115 | public static ProtocolDetector buildProtocolDetector(Recorder recorder) { 116 | return new ProtocolDetector( 117 | recorder, 118 | (ctx, buf) -> { 119 | recorder.recordEvent("unsupported protocol:" + NettyUtil.formatByteBuf(ctx, "detect", buf)); 120 | buf.release(); 121 | ctx.channel().writeAndFlush(Unpooled.wrappedBuffer(UN_SUPPORT_PROTOCOL_MSG)) 122 | .addListener(ChannelFutureListener.CLOSE); 123 | }, 124 | new HttpProxyMatcher(), 125 | new HttpsProxyMatcher(), 126 | new Socks5ProxyMatcher() 127 | ); 128 | } 129 | 130 | public void destroy() { 131 | NettyUtil.closeIfActive(serverChannel); 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/RuntimeIpSource.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy; 2 | 3 | import cn.iinti.proxycompose.Settings; 4 | import cn.iinti.proxycompose.resource.IpResourceParser; 5 | import cn.iinti.proxycompose.loop.Looper; 6 | import cn.iinti.proxycompose.proxy.outbound.IpPool; 7 | import cn.iinti.proxycompose.proxy.outbound.downloader.IpDownloader; 8 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 9 | import cn.iinti.proxycompose.trace.Recorder; 10 | import cn.iinti.proxycompose.trace.impl.SubscribeRecorders; 11 | import com.google.common.base.Splitter; 12 | import com.google.common.collect.Lists; 13 | import lombok.Getter; 14 | import lombok.experimental.Delegate; 15 | import org.apache.commons.lang3.StringUtils; 16 | 17 | import java.util.Collections; 18 | import java.util.List; 19 | 20 | public class RuntimeIpSource { 21 | @Delegate 22 | private final Settings.IpSource ipSource; 23 | 24 | /** 25 | * 代理IP池，真正存储IP资源，以及缓存的可用tcp连接 26 | */ 27 | @Getter 28 | private final IpPool ipPool; 29 | 30 | /** 31 | * 从代理源加载ip资源的下载器 32 | */ 33 | private final IpDownloader ipDownloader; 34 | 35 | /** 36 | * 绑定ip池的线程，使用单线程模型完成资源分发 37 | */ 38 | @Getter 39 | private final Looper looper; 40 | 41 | @Getter 42 | private final Recorder recorder; 43 | /** 44 | * 代理资源文本解析器 45 | */ 46 | @Getter 47 | private final IpResourceParser resourceParser; 48 | 49 | public RuntimeIpSource(Settings.IpSource ipSource) { 50 | this.ipSource = ipSource; 51 | this.supportProtocolList = Collections.unmodifiableList(parseSupportProtocol(ipSource.supportProtocol.value)); 52 | String sourceKey = ipSource.name; 53 | String beanId = "IpSource-" + sourceKey; 54 | 55 | looper = new Looper(beanId).startLoop(); 56 | recorder = SubscribeRecorders.IP_SOURCE.acquireRecorder(beanId, Settings.global.debug.value, sourceKey); 57 | ipPool = new IpPool(RuntimeIpSource.this, looper, recorder, sourceKey); 58 | ipDownloader = new IpDownloader(RuntimeIpSource.this, recorder, sourceKey); 59 | resourceParser = IpResourceParser.resolve(ipSource.resourceFormat.value); 60 | 61 | validCheck(); 62 | 63 | // 这里会启动下载任务，所以最后执行 64 | looper.scheduleWithRate(RuntimeIpSource.this::scheduleIpDownload, 65 | ipSource.reloadInterval.value * 1000); 66 | 67 | looper.scheduleWithRate(RuntimeIpSource.this::scheduleMakeConnCache, 68 | ipSource.makeConnInterval.value * 1000); 69 | 70 | looper.postDelay(RuntimeIpSource.this::scheduleIpDownload, 500); 71 | } 72 | 73 | void validCheck() { 74 | //todo 75 | } 76 | 77 | /** 78 | * 本IP池支持的代理协议：http/https/socks5 79 | */ 80 | @Getter 81 | private final List supportProtocolList; 82 | 83 | 84 | public boolean needAuth() { 85 | return StringUtils.isNoneBlank( 86 | ipSource.getUpstreamAuthUser().value, 87 | ipSource.getUpstreamAuthPassword().value 88 | ); 89 | } 90 | 91 | private static final Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); 92 | 93 | public static List parseSupportProtocol(String config) { 94 | List protocols = Lists.newArrayList(); 95 | for (String protocolStr : splitter.split(config)) { 96 | Protocol protocol = Protocol.get(protocolStr); 97 | if (protocol == null) { 98 | throw new IllegalArgumentException("error support protocol config:" + protocolStr); 99 | } 100 | protocols.add(protocol); 101 | } 102 | protocols.sort((o1, o2) -> Integer.compare(o2.getPriority(), o1.getPriority())); 103 | return protocols; 104 | } 105 | 106 | public void recordComposedEvent(Recorder userTraceRecorder, Recorder.MessageGetter messageGetter) { 107 | userTraceRecorder.recordEvent(messageGetter); 108 | recorder.recordEvent(messageGetter); 109 | } 110 | 111 | public void recordComposedMosaicEvent(Recorder userTraceRecorder, Recorder.MessageGetter messageGetter) { 112 | userTraceRecorder.recordMosaicMsgIfSubscribeRecorder(messageGetter); 113 | recorder.recordEvent(messageGetter); 114 | } 115 | 116 | 117 | private void scheduleIpDownload() { 118 | looper.checkLooper(); 119 | recorder.recordEvent("begin download ip"); 120 | ipDownloader.downloadIp(); 121 | } 122 | 123 | public void scheduleMakeConnCache() { 124 | ipPool.makeCache(); 125 | } 126 | 127 | public boolean poolEmpty() { 128 | return getIpPool().poolEmpty(); 129 | } 130 | public double healthScore() { 131 | return getIpPool().healthScore(); 132 | } 133 | 134 | public void destroy() { 135 | looper.execute(() -> { 136 | ipPool.destroy(); 137 | looper.postDelay(looper::close, 30_000); 138 | }); 139 | } 140 | 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/Session.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy; 2 | 3 | import cn.iinti.proxycompose.Settings; 4 | import cn.iinti.proxycompose.proxy.inbound.handlers.RelayHandler; 5 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 6 | import cn.iinti.proxycompose.resource.IpAndPort; 7 | import cn.iinti.proxycompose.utils.ConsistentHashUtil; 8 | import cn.iinti.proxycompose.trace.impl.SubscribeRecorders; 9 | import io.netty.channel.Channel; 10 | import io.netty.util.AttributeKey; 11 | import lombok.Getter; 12 | import lombok.Setter; 13 | 14 | import java.util.UUID; 15 | 16 | public class Session { 17 | 18 | @Getter 19 | private final String sessionId = UUID.randomUUID().toString(); 20 | 21 | @Getter 22 | private final SubscribeRecorders.SubscribeRecorder recorder = SubscribeRecorders.USER_SESSION 23 | .acquireRecorder(sessionId, Settings.global.debug.value, "default"); 24 | 25 | @Getter 26 | private final Channel inboundChannel; 27 | 28 | @Getter 29 | private final ProxyServer proxyServer; 30 | 31 | @Getter 32 | @Setter 33 | private boolean authed = false; 34 | 35 | 36 | public static Session touch(Channel inboundChannel, ProxyServer proxyServer) { 37 | return new Session(inboundChannel, proxyServer); 38 | } 39 | 40 | private Session(Channel inboundChannel, ProxyServer proxyServer) { 41 | this.inboundChannel = inboundChannel; 42 | this.proxyServer = proxyServer; 43 | 44 | recorder.recordEvent("new request from: " + inboundChannel); 45 | attach(inboundChannel); 46 | inboundChannel.closeFuture().addListener(future -> recorder.recordEvent("user connection closed")); 47 | } 48 | 49 | private static final AttributeKey SESSION_ATTRIBUTE_KEY = AttributeKey.newInstance("SESSION_ATTRIBUTE_KEY"); 50 | 51 | private void attach(Channel channel) { 52 | channel.attr(SESSION_ATTRIBUTE_KEY).set(this); 53 | } 54 | 55 | public static Session get(Channel channel) { 56 | return channel.attr(SESSION_ATTRIBUTE_KEY).get(); 57 | } 58 | 59 | 60 | public void replay(Channel upstreamChannel) { 61 | upstreamChannel.pipeline() 62 | .addLast(new RelayHandler(inboundChannel, "replay-outbound:", recorder)); 63 | 64 | inboundChannel.pipeline() 65 | .addLast(new RelayHandler(upstreamChannel, "replay-inbound:", recorder)); 66 | } 67 | 68 | 69 | // 代理的最终目标，从代理请求中提取，可以理解为一定存在（只要是合法的代理请求一定能解析到） 70 | @Getter 71 | private IpAndPort connectTarget; 72 | 73 | @Getter 74 | private Long sessionHash; 75 | @Getter 76 | private Protocol inboundProtocol; 77 | 78 | 79 | public void onProxyTargetResolved(IpAndPort ipAndPort, Protocol inboundProtocol) { 80 | String sessionIdKey = Settings.global.randomTurning.value ? 81 | String.valueOf(System.currentTimeMillis()) + getProxyServer().getPort() : 82 | String.valueOf(getProxyServer().getPort()); 83 | 84 | this.sessionHash = ConsistentHashUtil.murHash(sessionIdKey); 85 | 86 | this.connectTarget = ipAndPort; 87 | this.inboundProtocol = inboundProtocol; 88 | 89 | 90 | recorder.recordEvent(() -> "proxy target resolved -> \nsessionHash: " + sessionHash 91 | + "\ntarget:" + ipAndPort 92 | + "\ninbound protocol: " + inboundProtocol 93 | + "\n" 94 | ); 95 | } 96 | 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/detector/HttpProxyMatcher.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.detector; 2 | 3 | 4 | import cn.iinti.proxycompose.proxy.inbound.handlers.ProxyHttpHandler; 5 | import cn.iinti.proxycompose.trace.Recorder; 6 | import com.google.common.collect.Sets; 7 | import io.netty.buffer.ByteBuf; 8 | import io.netty.channel.ChannelHandlerContext; 9 | import io.netty.handler.codec.http.HttpServerCodec; 10 | 11 | import java.util.Set; 12 | 13 | import static java.nio.charset.StandardCharsets.US_ASCII; 14 | 15 | /** 16 | * Matcher for plain http request. 17 | */ 18 | public class HttpProxyMatcher extends ProtocolMatcher { 19 | 20 | 21 | private static final Set methods = Sets.newHashSet("GET", "POST", "PUT", "HEAD", "OPTIONS", "PATCH", "DELETE", 22 | "TRACE"); 23 | 24 | 25 | @Override 26 | public int match(ByteBuf buf) { 27 | if (buf.readableBytes() < 8) { 28 | return PENDING; 29 | } 30 | 31 | int index = buf.indexOf(0, 8, (byte) ' '); 32 | if (index < 0) { 33 | return MISMATCH; 34 | } 35 | 36 | int firstURIIndex = index + 1; 37 | if (buf.readableBytes() < firstURIIndex + 1) { 38 | return PENDING; 39 | } 40 | 41 | String method = buf.toString(0, index, US_ASCII); 42 | char firstURI = (char) (buf.getByte(firstURIIndex + buf.readerIndex()) & 0xff); 43 | if (!methods.contains(method) || firstURI == '/') { 44 | return MISMATCH; 45 | } 46 | 47 | return MATCH; 48 | } 49 | 50 | @Override 51 | protected void handleMatched( Recorder recorder, ChannelHandlerContext ctx) { 52 | recorder.recordEvent("new mock http request "); 53 | ctx.pipeline().addLast( 54 | new HttpServerCodec(), 55 | new ProxyHttpHandler(recorder, false) 56 | ); 57 | } 58 | 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/detector/HttpsProxyMatcher.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.detector; 2 | 3 | 4 | import cn.iinti.proxycompose.proxy.inbound.handlers.ProxyHttpHandler; 5 | import cn.iinti.proxycompose.trace.Recorder; 6 | import io.netty.buffer.ByteBuf; 7 | import io.netty.channel.ChannelHandlerContext; 8 | import io.netty.handler.codec.http.HttpServerCodec; 9 | 10 | import static java.nio.charset.StandardCharsets.US_ASCII; 11 | 12 | /** 13 | * Matcher for http proxy connect tunnel. 14 | */ 15 | public class HttpsProxyMatcher extends ProtocolMatcher { 16 | 17 | @Override 18 | public int match(ByteBuf buf) { 19 | if (buf.readableBytes() < 8) { 20 | return PENDING; 21 | } 22 | 23 | String method = buf.toString(0, 8, US_ASCII); 24 | if (!"CONNECT ".equalsIgnoreCase(method)) { 25 | return MISMATCH; 26 | } 27 | 28 | return MATCH; 29 | } 30 | 31 | 32 | @Override 33 | protected void handleMatched(Recorder recorder, ChannelHandlerContext ctx) { 34 | ctx.pipeline().addLast( 35 | new HttpServerCodec(), 36 | new ProxyHttpHandler(recorder, true) 37 | ); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/detector/ProtocolDetector.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.detector; 2 | 3 | 4 | import cn.iinti.proxycompose.utils.NettyUtil; 5 | import cn.iinti.proxycompose.trace.Recorder; 6 | import io.netty.buffer.ByteBuf; 7 | import io.netty.channel.ChannelHandlerContext; 8 | import io.netty.channel.ChannelInboundHandlerAdapter; 9 | import io.netty.handler.codec.ByteToMessageDecoder; 10 | import io.netty.util.ReferenceCountUtil; 11 | import lombok.extern.slf4j.Slf4j; 12 | 13 | import java.io.IOException; 14 | 15 | import static io.netty.handler.codec.ByteToMessageDecoder.MERGE_CUMULATOR; 16 | 17 | /** 18 | * Switcher to distinguish different protocols 19 | */ 20 | @Slf4j 21 | public class ProtocolDetector extends ChannelInboundHandlerAdapter { 22 | 23 | private final ByteToMessageDecoder.Cumulator cumulator = MERGE_CUMULATOR; 24 | private final ProtocolMatcher[] matcherList; 25 | 26 | private ByteBuf buf; 27 | 28 | private final MatchMissHandler missHandler; 29 | 30 | private final Recorder recorder; 31 | private boolean hasData = false; 32 | 33 | 34 | public ProtocolDetector( Recorder recorder, MatchMissHandler missHandler, ProtocolMatcher... matchers) { 35 | this.recorder = recorder; 36 | this.missHandler = missHandler; 37 | if (matchers.length == 0) { 38 | throw new IllegalArgumentException("No matcher for ProtocolDetector"); 39 | } 40 | this.matcherList = matchers; 41 | } 42 | 43 | @Override 44 | public void channelRead(ChannelHandlerContext ctx, Object msg) { 45 | if (!(msg instanceof ByteBuf)) { 46 | ReferenceCountUtil.release(msg); 47 | recorder.recordEvent("unexpected message type for ProtocolDetector: " + msg.getClass()); 48 | NettyUtil.closeOnFlush(ctx.channel()); 49 | return; 50 | } 51 | hasData = true; 52 | 53 | ByteBuf in = (ByteBuf) msg; 54 | if (buf == null) { 55 | buf = in; 56 | } else { 57 | buf = cumulator.cumulate(ctx.alloc(), buf, in); 58 | } 59 | // the buf maybe null after recorder acquire variable 60 | ByteBuf tmpBuf = buf; 61 | recorder.recordEvent(() -> "begin protocol detect with header: " + tmpBuf.readableBytes()); 62 | boolean hasPending = false; 63 | for (ProtocolMatcher matcher : matcherList) { 64 | int match = matcher.match(buf.duplicate()); 65 | if (match == ProtocolMatcher.MATCH) { 66 | recorder.recordEvent("matched by " + matcher.getClass().getName()); 67 | matcher.handleMatched( recorder, ctx); 68 | ctx.pipeline().remove(this); 69 | ctx.fireChannelRead(buf); 70 | return; 71 | } 72 | 73 | if (match == ProtocolMatcher.PENDING) { 74 | recorder.recordEvent("match " + matcher.getClass() + " pending.."); 75 | hasPending = true; 76 | } 77 | } 78 | if (hasPending) { 79 | return; 80 | } 81 | 82 | // all miss 83 | missHandler.onAllMatchMiss(ctx, buf); 84 | buf = null; 85 | } 86 | 87 | @Override 88 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 89 | if (buf != null) { 90 | buf.release(); 91 | buf = null; 92 | } 93 | if (!hasData && cause instanceof IOException) { 94 | // 有LBS负载均衡的服务，通过探测端口是否开启来判断服务是否存活， 95 | // 他们只开启端口，然后就会关闭隧道，此时这里就会有IOException: java.io.IOException: Connection reset by peer 96 | recorder.recordEvent(() -> "exception: " + cause.getClass() + " ->" + cause.getMessage() + " before any data receive"); 97 | } else { 98 | recorder.recordEvent("protocol detect error", cause); 99 | } 100 | NettyUtil.closeOnFlush(ctx.channel()); 101 | } 102 | 103 | public interface MatchMissHandler { 104 | void onAllMatchMiss(ChannelHandlerContext ctx, ByteBuf buf); 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/detector/ProtocolMatcher.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.detector; 2 | 3 | 4 | import cn.iinti.proxycompose.trace.Recorder; 5 | import io.netty.buffer.ByteBuf; 6 | import io.netty.channel.ChannelHandlerContext; 7 | 8 | /** 9 | * Matcher for protocol. 10 | */ 11 | public abstract class ProtocolMatcher { 12 | 13 | static int MATCH = 1; 14 | static int MISMATCH = -1; 15 | static int PENDING = 0; 16 | 17 | /** 18 | * If match the protocol. 19 | * 20 | * @return 1:match, -1:not match, 0:still can not judge now 21 | */ 22 | protected abstract int match(ByteBuf buf); 23 | 24 | /** 25 | * Deal with the pipeline when matched 26 | */ 27 | protected void handleMatched(Recorder recorder, ChannelHandlerContext ctx) { 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/detector/Socks5ProxyMatcher.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.detector; 2 | 3 | 4 | import cn.iinti.proxycompose.proxy.inbound.handlers.ProxySocks5Handler; 5 | import cn.iinti.proxycompose.trace.Recorder; 6 | import io.netty.buffer.ByteBuf; 7 | import io.netty.channel.ChannelHandlerContext; 8 | import io.netty.handler.codec.socks.SocksInitRequestDecoder; 9 | import io.netty.handler.codec.socks.SocksMessageEncoder; 10 | 11 | /** 12 | * Matcher for socks5 proxy protocol 13 | */ 14 | public class Socks5ProxyMatcher extends ProtocolMatcher { 15 | 16 | 17 | @Override 18 | public int match(ByteBuf buf) { 19 | if (buf.readableBytes() < 2) { 20 | return PENDING; 21 | } 22 | byte first = buf.getByte(buf.readerIndex()); 23 | byte second = buf.getByte(buf.readerIndex() + 1); 24 | if (first == 5) { 25 | return MATCH; 26 | } 27 | return MISMATCH; 28 | } 29 | 30 | 31 | @Override 32 | protected void handleMatched(Recorder recorder, ChannelHandlerContext ctx) { 33 | ctx.pipeline().addLast( 34 | new SocksInitRequestDecoder(), 35 | new SocksMessageEncoder(), 36 | new ProxySocks5Handler() 37 | ); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/handlers/ProxySocks5Handler.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.handlers; 2 | 3 | 4 | import cn.iinti.proxycompose.Settings; 5 | import cn.iinti.proxycompose.resource.IpAndPort; 6 | import cn.iinti.proxycompose.proxy.Session; 7 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 8 | import cn.iinti.proxycompose.proxy.switcher.OutboundConnectTask; 9 | import cn.iinti.proxycompose.proxy.switcher.UpstreamHandSharkCallback; 10 | import cn.iinti.proxycompose.utils.NettyUtil; 11 | import cn.iinti.proxycompose.utils.IpUtils; 12 | import cn.iinti.proxycompose.trace.Recorder; 13 | import io.netty.channel.*; 14 | import io.netty.handler.codec.socks.*; 15 | 16 | import java.util.List; 17 | 18 | public class ProxySocks5Handler extends SimpleChannelInboundHandler 19 | implements UpstreamHandSharkCallback { 20 | private ChannelHandlerContext ctx; 21 | private SocksCmdRequest req; 22 | private Session session; 23 | private Recorder recorder; 24 | 25 | 26 | @Override 27 | protected void channelRead0(ChannelHandlerContext ctx, SocksRequest socksRequest) { 28 | this.ctx = ctx; 29 | if (this.session == null) { 30 | this.session = Session.get(ctx.channel()); 31 | this.recorder = session.getRecorder(); 32 | } 33 | recorder.recordEvent(() -> "inbound handle socks request：" + socksRequest.requestType()); 34 | switch (socksRequest.requestType()) { 35 | case INIT: 36 | handleInit((SocksInitRequest) socksRequest); 37 | break; 38 | case AUTH: 39 | handleAuth(ctx, (SocksAuthRequest) socksRequest); 40 | break; 41 | case CMD: 42 | handleCmd(ctx, socksRequest); 43 | break; 44 | case UNKNOWN: 45 | default: 46 | ctx.close(); 47 | } 48 | } 49 | 50 | private void handleAuth(ChannelHandlerContext ctx, SocksAuthRequest socksRequest) { 51 | if (session.isAuthed()) { 52 | ctx.pipeline().addFirst(new SocksCmdRequestDecoder()); 53 | ctx.writeAndFlush(new SocksAuthResponse(SocksAuthStatus.SUCCESS)); 54 | return; 55 | } 56 | 57 | String username = socksRequest.username(); 58 | String password = socksRequest.password(); 59 | session.setAuthed(Settings.global.authRules.doAuth(username, password)); 60 | 61 | if (session.isAuthed()) { 62 | ctx.pipeline().addFirst(new SocksCmdRequestDecoder()); 63 | ctx.writeAndFlush(new SocksAuthResponse(SocksAuthStatus.SUCCESS)); 64 | return; 65 | } 66 | 67 | ctx.pipeline().addFirst(new SocksAuthRequestDecoder()); 68 | ctx.writeAndFlush(new SocksAuthResponse(SocksAuthStatus.FAILURE)); 69 | } 70 | 71 | 72 | private void handleInit(SocksInitRequest socksInitRequest) { 73 | // 对于socks代理来说，ip鉴权和密码鉴权是两个步骤，所以需要提前判定,先鉴权IP，然后鉴权密码 74 | if (session.isAuthed()) { 75 | recorder.recordEvent(() -> "has been authed"); 76 | ctx.pipeline().addFirst(new SocksCmdRequestDecoder()); 77 | ctx.writeAndFlush(new SocksInitResponse(SocksAuthScheme.NO_AUTH)); 78 | return; 79 | } 80 | 81 | List socksAuthSchemes = socksInitRequest.authSchemes(); 82 | for (SocksAuthScheme socksAuthScheme : socksAuthSchemes) { 83 | if (socksAuthScheme == SocksAuthScheme.AUTH_PASSWORD) { 84 | // 如果客户端明确说明支持密码鉴权，那么要求提供密码 85 | ctx.pipeline().addFirst(new SocksAuthRequestDecoder()); 86 | ctx.writeAndFlush(new SocksInitResponse(SocksAuthScheme.AUTH_PASSWORD)); 87 | return; 88 | } 89 | 90 | } 91 | 92 | recorder.recordEvent(() -> "require password auth"); 93 | ctx.pipeline().addFirst(new SocksAuthRequestDecoder()); 94 | ctx.writeAndFlush(new SocksInitResponse(SocksAuthScheme.AUTH_PASSWORD)); 95 | 96 | } 97 | 98 | private void handleCmd(ChannelHandlerContext ctx, SocksRequest socksRequest) { 99 | req = (SocksCmdRequest) socksRequest; 100 | //TODO 实现UDP转发 101 | if (req.cmdType() != SocksCmdType.CONNECT) { 102 | recorder.recordEvent(() -> "not support s5 cmd: " + req.cmdType()); 103 | ctx.close(); 104 | return; 105 | } 106 | session.onProxyTargetResolved(new IpAndPort(req.host(), req.port()), Protocol.SOCKS5); 107 | 108 | OutboundConnectTask.startConnectOutbound(session, this); 109 | } 110 | 111 | 112 | @Override 113 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 114 | recorder.recordEvent(() -> "Socks5InboundHandler handler error", cause); 115 | } 116 | 117 | @Override 118 | public void onHandSharkFinished(Channel upstreamChannel, Protocol outboundProtocol) { 119 | NettyUtil.loveOther(session.getInboundChannel(), upstreamChannel); 120 | ctx.channel().eventLoop().execute(() -> { 121 | IpAndPort connectTarget = session.getConnectTarget(); 122 | SocksCmdResponse socksCmdResponse = new SocksCmdResponse(SocksCmdStatus.SUCCESS, 123 | IpUtils.isIpV4(connectTarget.getIp()) ? SocksAddressType.IPv4 : SocksAddressType.DOMAIN, 124 | connectTarget.getIp(), connectTarget.getPort() 125 | ); 126 | ctx.channel().writeAndFlush(socksCmdResponse) 127 | .addListener(future -> { 128 | if (!future.isSuccess()) { 129 | recorder.recordEvent(() -> "socket closed when write socks success", future.cause()); 130 | return; 131 | } 132 | ChannelPipeline pipeline = ctx.pipeline(); 133 | pipeline.remove(SocksMessageEncoder.class); 134 | pipeline.remove(ProxySocks5Handler.class); 135 | recorder.recordEvent(() -> "start socks5 replay tuning"); 136 | session.replay(upstreamChannel); 137 | }); 138 | }); 139 | 140 | } 141 | 142 | @Override 143 | public void onHandSharkError(Throwable e) { 144 | ctx.channel().writeAndFlush(new SocksCmdResponse(SocksCmdStatus.FAILURE, req.addressType())) 145 | .addListener(ChannelFutureListener.CLOSE); 146 | } 147 | 148 | } -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/inbound/handlers/RelayHandler.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.inbound.handlers; 2 | 3 | 4 | import cn.iinti.proxycompose.trace.Recorder; 5 | import io.netty.buffer.Unpooled; 6 | import io.netty.channel.Channel; 7 | import io.netty.channel.ChannelHandlerContext; 8 | import io.netty.channel.ChannelInboundHandlerAdapter; 9 | import io.netty.util.ReferenceCountUtil; 10 | 11 | public final class RelayHandler extends ChannelInboundHandlerAdapter { 12 | 13 | private final Channel nextChannel; 14 | private final String TAG; 15 | 16 | private final Recorder recorder; 17 | 18 | public RelayHandler(Channel relayChannel, String tag, Recorder recorder) { 19 | this.nextChannel = relayChannel; 20 | this.TAG = tag; 21 | this.recorder = recorder; 22 | } 23 | 24 | @Override 25 | public void channelActive(ChannelHandlerContext ctx) { 26 | ctx.writeAndFlush(Unpooled.EMPTY_BUFFER); 27 | } 28 | 29 | @Override 30 | public void channelInactive(ChannelHandlerContext ctx) throws Exception { 31 | recorder.recordEvent("channel channelInactive"); 32 | super.channelInactive(ctx); 33 | } 34 | 35 | @Override 36 | public void channelRead(ChannelHandlerContext ctx, Object msg) { 37 | recorder.recordEvent(TAG + ": receive message: " + msg); 38 | if (nextChannel.isActive()) { 39 | nextChannel.writeAndFlush(msg); 40 | } else { 41 | ReferenceCountUtil.release(msg); 42 | } 43 | } 44 | 45 | 46 | @Override 47 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 48 | recorder.recordEvent(() -> TAG + ": replay exception", cause); 49 | if (nextChannel.isActive()) { 50 | nextChannel.write(Unpooled.EMPTY_BUFFER) 51 | .addListener(future -> nextChannel.close()); 52 | } else { 53 | ctx.close(); 54 | } 55 | 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/ActiveProxyIp.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound; 2 | 3 | import cn.iinti.proxycompose.proxy.outbound.downloader.DownloadProxyIp; 4 | import cn.iinti.proxycompose.utils.ConsistentHashUtil; 5 | import cn.iinti.proxycompose.utils.NettyUtil; 6 | import cn.iinti.proxycompose.resource.DropReason; 7 | import cn.iinti.proxycompose.loop.Looper; 8 | import cn.iinti.proxycompose.loop.ValueCallback; 9 | import cn.iinti.proxycompose.trace.Recorder; 10 | import com.google.common.collect.Lists; 11 | import com.google.common.io.BaseEncoding; 12 | import io.netty.channel.Channel; 13 | import io.netty.util.AttributeKey; 14 | import lombok.Getter; 15 | import org.apache.commons.lang3.StringUtils; 16 | 17 | import java.lang.ref.WeakReference; 18 | import java.nio.charset.StandardCharsets; 19 | import java.util.HashSet; 20 | import java.util.LinkedList; 21 | import java.util.Set; 22 | import java.util.concurrent.ConcurrentHashMap; 23 | 24 | public class ActiveProxyIp { 25 | 26 | @Getter 27 | private final DownloadProxyIp downloadProxyIp; 28 | 29 | private final Looper workThread; 30 | private final Recorder recorder; 31 | @Getter 32 | private final IpPool ipPool; 33 | 34 | @Getter 35 | private final long murHash; 36 | @Getter 37 | private final long seq; 38 | 39 | @Getter 40 | private volatile ActiveStatus activeStatus; 41 | 42 | 43 | private final Set usedChannels = ConcurrentHashMap.newKeySet(); 44 | 45 | private final CacheHandle cachedHandle = new CacheHandle(); 46 | 47 | private final Set usedHash = new HashSet<>(); 48 | 49 | 50 | private Runnable destroyFun; 51 | 52 | public ActiveProxyIp(IpPool ipPool, DownloadProxyIp downloadProxyIp, long seq) { 53 | this.ipPool = ipPool; 54 | this.seq = seq; 55 | this.downloadProxyIp = downloadProxyIp; 56 | this.workThread = ipPool.getRuntimeIpSource().getLooper(); 57 | this.recorder = ipPool.getRuntimeIpSource().getRecorder(); 58 | this.activeStatus = ActiveStatus.ONLINE; 59 | this.murHash = ConsistentHashUtil.murHash(downloadProxyIp.getResourceId()); 60 | } 61 | 62 | public void destroy(DropReason dropReason) { 63 | if (activeStatus == ActiveStatus.DESTROY) { 64 | return; 65 | } 66 | workThread.execute(() -> { 67 | if (activeStatus == ActiveStatus.DESTROY) { 68 | return; 69 | } 70 | if (activeStatus == ActiveStatus.ONLINE) { 71 | activeStatus = ActiveStatus.OFFLINE; 72 | } 73 | 74 | destroyFun = () -> { 75 | workThread.checkLooper(); 76 | if (activeStatus == ActiveStatus.DESTROY) { 77 | return; 78 | } 79 | activeStatus = ActiveStatus.DESTROY; 80 | destroyFun = null; 81 | cachedHandle.destroy(); 82 | }; 83 | if (usedHash.isEmpty() || dropReason == DropReason.IP_SERVER_UNAVAILABLE) { 84 | // 如果当前没有用户占用本ip，则立即销毁 85 | destroyFun.run(); 86 | return; 87 | } 88 | // 否则给一个60s的延时时间，给业务继续使用 89 | // 请注意整个60s的延时不是确定的，如果业务提前判断全部离开本ip，则清理动作可以提前 90 | workThread.postDelay(destroyFun, 60_000); 91 | }); 92 | 93 | 94 | } 95 | 96 | 97 | public void borrowConnect(Recorder recorder, String tag, 98 | ActivityProxyIpBindObserver observer, 99 | ValueCallback valueCallback) { 100 | observer.onBind(this); 101 | workThread.execute(() -> { 102 | cachedHandle.onConnBorrowed(); 103 | 104 | // 尝试使用缓存的ip资源 105 | while (true) { 106 | Channel one = cachedHandle.cachedChannels.poll(); 107 | if (one == null) { 108 | break; 109 | } 110 | if (one.isActive()) { 111 | recorder.recordEvent(() -> tag + "conn cache pool hinted"); 112 | ValueCallback.success(valueCallback, one); 113 | return; 114 | } 115 | } 116 | recorder.recordEvent(() -> tag + "begin to create connection immediately"); 117 | 118 | createUpstreamConnection(valueCallback, recorder); 119 | 120 | }); 121 | 122 | } 123 | 124 | public boolean isIdle() { 125 | return usedChannels.isEmpty(); 126 | } 127 | 128 | private static final AttributeKey ACTIVITY_PROXY_IP_KEY = AttributeKey.newInstance("ACTIVITY_PROXY_IP"); 129 | 130 | private void createUpstreamConnection(ValueCallback valueCallback, Recorder userRecorder) { 131 | OutboundOperator.connectToServer(downloadProxyIp.getProxyHost(), downloadProxyIp.getProxyPort(), value -> { 132 | if (!value.isSuccess()) { 133 | // 这里失败我们我们不再执行立即替换出口ip的逻辑，这是因为在并发极高的情况下 134 | // 失败可能是我们自己的网络不通畅导致的，我们不能以链接失败就判定ip存在问题 135 | ipPool.onCreateConnectionFailed(ActiveProxyIp.this, value.e, userRecorder); 136 | ValueCallback.failed(valueCallback, value.e); 137 | return; 138 | } 139 | 140 | // setup meta info 141 | Channel channel = value.v; 142 | channel.attr(ACTIVITY_PROXY_IP_KEY).set(this); 143 | channel.closeFuture().addListener(it -> usedChannels.remove(channel)); 144 | cachedHandle.scheduleCleanIdleCache(channel, ipPool.getRuntimeIpSource().getConnIdleSeconds().value); 145 | 146 | ValueCallback.success(valueCallback, channel); 147 | }); 148 | } 149 | 150 | public static void restoreCache(Channel channel) { 151 | ActiveProxyIp activeProxyIp = getBinding(channel); 152 | if (activeProxyIp == null) { 153 | channel.close(); 154 | return; 155 | } 156 | activeProxyIp.cachedHandle.restoreCache(channel); 157 | } 158 | 159 | public static void offlineBindingProxy(Channel channel, IpPool.OfflineLevel level, Recorder userRecorder) { 160 | ActiveProxyIp activeProxyIp = getBinding(channel); 161 | if (activeProxyIp == null) { 162 | return; 163 | } 164 | activeProxyIp.getIpPool().offlineProxy(activeProxyIp, level, userRecorder); 165 | } 166 | 167 | public static ActiveProxyIp getBinding(Channel channel) { 168 | return channel.attr(ACTIVITY_PROXY_IP_KEY).get(); 169 | } 170 | 171 | public void makeCache() { 172 | if (activeStatus != ActiveStatus.ONLINE) { 173 | return; 174 | } 175 | cachedHandle.doCreateCacheTask(); 176 | } 177 | 178 | private class CacheHandle { 179 | private final LinkedList cachedChannels = Lists.newLinkedList(); 180 | /** 181 | * 用户请求的平均时间间隔，代表这个ip资源处理请求的qps，但是我们使用一个高效计算方案评估这个指标
182 | * 这个值将会约等于最近10次请求时间间隔的平均数 183 | */ 184 | private double avgInterval = 1000; 185 | 186 | private long lastRequestConnection = 0; 187 | 188 | private long lastCreateCacheConnection = 0; 189 | 190 | private void destroy() { 191 | NettyUtil.closeAll(cachedChannels); 192 | cachedChannels.clear(); 193 | } 194 | 195 | public void restoreCache(Channel channel) { 196 | workThread.execute(() -> { 197 | if (activeStatus == ActiveStatus.DESTROY) { 198 | channel.close(); 199 | return; 200 | } 201 | cachedChannels.addFirst(channel); 202 | }); 203 | } 204 | 205 | public void onConnBorrowed() { 206 | long now = System.currentTimeMillis(); 207 | if (lastRequestConnection == 0L) { 208 | lastRequestConnection = now; 209 | return; 210 | } 211 | 212 | long thisInterval = now - lastRequestConnection; 213 | lastRequestConnection = now; 214 | avgInterval = (thisInterval * 9 + avgInterval) / 10; 215 | } 216 | 217 | void scheduleCleanIdleCache(Channel cacheChannel, Integer idleSeconds) { 218 | // 为了避免gc hold，所有延时任务里面不允许直接访问channel对象，而使用WeakReference 219 | WeakReference channelRef = new WeakReference<>(cacheChannel); 220 | workThread.postDelay(() -> { 221 | Channel gcChannel = channelRef.get(); 222 | if (gcChannel == null || !gcChannel.isActive()) { 223 | return; 224 | } 225 | for (Channel ch : cachedChannels) { 226 | if (ch.equals(gcChannel)) { 227 | gcChannel.close(); 228 | return; 229 | } 230 | } 231 | }, idleSeconds * 1000); 232 | } 233 | 234 | 235 | public void doCreateCacheTask() { 236 | if (cachedChannels.size() > 3) { 237 | recorder.recordEvent(() -> "skip make conn cache because of cache overflow:" + cachedChannels.size()); 238 | //理论上真实流量一半的速率，cachedChannels.size()应该为0或者1 239 | return; 240 | } 241 | 242 | long now = System.currentTimeMillis(); 243 | if (lastCreateCacheConnection != 0 && (now - lastRequestConnection) < avgInterval / 2) { 244 | // 流量为真实速率的一半 245 | return; 246 | } 247 | lastCreateCacheConnection = now; 248 | 249 | recorder.recordEvent(() -> "fire conn cache make task"); 250 | createUpstreamConnection(value -> { 251 | if (value.isSuccess()) { 252 | workThread.execute(() -> cachedChannels.addFirst(value.v)); 253 | } 254 | }, Recorder.nop); 255 | } 256 | 257 | } 258 | 259 | public boolean canUserPassAuth() { 260 | return StringUtils.isNoneBlank( 261 | downloadProxyIp.getUserName(), 262 | downloadProxyIp.getPassword() 263 | ); 264 | } 265 | 266 | 267 | public String buildHttpAuthenticationInfo() { 268 | if (canUserPassAuth()) { 269 | String authorizationBody = downloadProxyIp.getUserName() + ":" + downloadProxyIp.getPassword(); 270 | return "Basic " + BaseEncoding.base64().encode(authorizationBody.getBytes(StandardCharsets.UTF_8)); 271 | } 272 | 273 | return null; 274 | } 275 | 276 | public void refreshRefSessionHash(long sessionHash, boolean add) { 277 | workThread.execute(() -> { 278 | if (add) { 279 | usedHash.add(sessionHash); 280 | return; 281 | } 282 | usedHash.remove(sessionHash); 283 | if (usedHash.isEmpty() && destroyFun != null) { 284 | destroyFun.run(); 285 | } 286 | }); 287 | } 288 | 289 | 290 | 291 | public enum ActiveStatus { 292 | ONLINE, 293 | OFFLINE, 294 | DESTROY 295 | } 296 | 297 | public interface ActivityProxyIpBindObserver { 298 | void onBind(ActiveProxyIp activeProxyIp); 299 | } 300 | 301 | 302 | } 303 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/IpPool.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound; 2 | 3 | import cn.iinti.proxycompose.resource.DropReason; 4 | import cn.iinti.proxycompose.loop.Looper; 5 | import cn.iinti.proxycompose.loop.ValueCallback; 6 | import cn.iinti.proxycompose.proxy.outbound.downloader.DownloadProxyIp; 7 | import cn.iinti.proxycompose.proxy.RuntimeIpSource; 8 | import cn.iinti.proxycompose.utils.ConsistentHashUtil; 9 | import cn.iinti.proxycompose.trace.Recorder; 10 | import com.alibaba.fastjson.JSONObject; 11 | import lombok.Getter; 12 | 13 | import java.lang.ref.WeakReference; 14 | import java.util.*; 15 | import java.util.concurrent.atomic.AtomicLong; 16 | import java.util.function.Function; 17 | 18 | public class IpPool { 19 | @Getter 20 | private final RuntimeIpSource runtimeIpSource; 21 | private final Looper workThread; 22 | private final Recorder recorder; 23 | 24 | /** 25 | * 一致性哈希ip池,提供在池中随机（固定session）的能力 26 | */ 27 | private final TreeMap poolWithMurHash = new TreeMap<>(); 28 | 29 | 30 | /** 31 | * 非ip池，一个独立的，用于track代理资源入库时间的存储域 32 | */ 33 | private final TreeMap poolWithCreateSequence = new TreeMap<>(); 34 | private final AtomicLong outboundInc = new AtomicLong(0); 35 | /** 36 | * 备用ip资源，溢出的额外代理ip资源 37 | */ 38 | private final LinkedHashMap cachedProxies = new LinkedHashMap() { 39 | @Override 40 | protected boolean removeEldestEntry(Map.Entry eldest) { 41 | // 自动删除超量ip，曾经因为没有处理这个变量，导致内存有两百万的ip资源 42 | // 范围为 10 - 1024之间的，配置ip池化大小的3倍容量 43 | Integer configPoolSize = runtimeIpSource.getPoolSize().value; 44 | int maxCacheSize = Math.min(configPoolSize * 3, 1024); 45 | maxCacheSize = Math.max(10, maxCacheSize); 46 | return size() > maxCacheSize; 47 | } 48 | }; 49 | 50 | public IpPool(RuntimeIpSource runtimeIpSource, Looper workThread, Recorder recorder, String sourceKey) { 51 | this.runtimeIpSource = runtimeIpSource; 52 | this.workThread = workThread; 53 | this.recorder = recorder; 54 | } 55 | 56 | public void allocateIp(long hash, Recorder userRecorder, 57 | ValueCallback valueCallback) { 58 | workThread.execute(() -> { 59 | if (poolWithMurHash.isEmpty()) { 60 | ValueCallback.failed(valueCallback, "no ip in pool"); 61 | return; 62 | } 63 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "allocateOutbound request" + 64 | " hash: " + hash + 65 | " pool size:" + poolWithMurHash.size() 66 | ); 67 | 68 | ActiveProxyIp activeProxyIp; 69 | 70 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "default session hash plan"); 71 | // 默认策略，在当前ip池中进行一致性哈希绑定 72 | activeProxyIp = ConsistentHashUtil.fetchConsistentRing(poolWithMurHash, hash); 73 | 74 | if (activeProxyIp == null) { 75 | ValueCallback.failed(valueCallback, "no suitable ip in the pool"); 76 | } else { 77 | ValueCallback.success(valueCallback, activeProxyIp); 78 | } 79 | }); 80 | } 81 | 82 | 83 | public void onCreateConnectionFailed(ActiveProxyIp activeProxyIp, Throwable cause, Recorder userRecorder) { 84 | runtimeIpSource.recordComposedEvent(userRecorder, () -> 85 | "连接创建失败->,进入IP资源下线决策阶段," 86 | + "当前系统缓存池大小:" + cachedProxies.size() 87 | ); 88 | runtimeIpSource.recordComposedMosaicEvent(userRecorder, cause::getMessage); 89 | 90 | 91 | String message = cause.getMessage(); 92 | OfflineLevel offlineLevel; 93 | String offlineReason = "未知原因"; 94 | if (message.contains("Connection refused") || message.contains("拒绝连接")) { 95 | // 明确这个ip资源无法使用了,强制下线 96 | offlineLevel = OfflineLevel.MUST; 97 | offlineReason = "拒绝连接"; 98 | } else if (message.contains("connection timed out") || message.contains("连接超时")) { 99 | // 连接超时，可能是防火墙拦截，也可能是对方服务器负载高无法处理我们的请求 100 | // 此时根据ip池的容量大小决定是否要下线 101 | offlineLevel = OfflineLevel.STRONG; 102 | offlineReason = "连接超时"; 103 | } else { 104 | // 其他失败原因，在ip池非常富裕的条件下执行下线 105 | offlineLevel = OfflineLevel.SUGGEST; 106 | } 107 | 108 | 109 | String finalOfflineReason = offlineReason; 110 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "资产下线触发强度：" + offlineLevel + " 下线原因:" + finalOfflineReason); 111 | 112 | offlineProxy(activeProxyIp, offlineLevel, userRecorder); 113 | } 114 | 115 | public boolean poolEmpty() { 116 | return poolWithMurHash.isEmpty(); 117 | } 118 | 119 | public double healthScore() { 120 | return (poolWithMurHash.size() + cachedProxies.size()) * 100.0 121 | / runtimeIpSource.getPoolSize().value; 122 | } 123 | 124 | public enum OfflineLevel { 125 | MUST("MUST"),// 必须下线，不关心什么原因 126 | STRONG("STRONG"),// 强烈要求下线，除非系统状态bad 127 | SUGGEST("SUGGEST")// 建议下线，轻微请求，除非系统非常富裕 128 | ; 129 | // 加一个name是为了避免混淆修改class的名字，毕竟添加keep很容易导致keep范围被放大 130 | public final String name; 131 | 132 | OfflineLevel(String name) { 133 | this.name = name; 134 | } 135 | } 136 | 137 | public void offlineProxy(ActiveProxyIp activeProxyIp, OfflineLevel level, Recorder userRecorder) { 138 | 139 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "offline proxy by pool status: " + level.name); 140 | runtimeIpSource.recordComposedMosaicEvent(userRecorder, () -> " proxy: " + activeProxyIp.getDownloadProxyIp().getResourceId()); 141 | if (poolWithMurHash.size() > runtimeIpSource.getPoolSize().value) { 142 | // ip池大小变更，直接下线 143 | offlineProxy(activeProxyIp, level == OfflineLevel.MUST ? DropReason.IP_SERVER_UNAVAILABLE : DropReason.IP_IDLE_POOL_OVERFLOW, userRecorder); 144 | return; 145 | } 146 | switch (level) { 147 | case MUST: 148 | offlineProxy(activeProxyIp, DropReason.IP_SERVER_UNAVAILABLE, userRecorder); 149 | return; 150 | case STRONG: 151 | if (!cachedProxies.isEmpty()) { 152 | offlineProxy(activeProxyIp, DropReason.IP_QUALITY_BAD, userRecorder); 153 | return; 154 | } 155 | break; 156 | case SUGGEST: 157 | if (cachedProxies.size() > runtimeIpSource.getPoolSize().value / 3) { 158 | offlineProxy(activeProxyIp, DropReason.IP_IDLE_POOL_OVERFLOW, userRecorder); 159 | return; 160 | } 161 | break; 162 | } 163 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "do not offline proxy finally"); 164 | } 165 | 166 | private void offlineProxy(ActiveProxyIp activeProxyIp, DropReason dropReason, Recorder userRecorder) { 167 | if (activeProxyIp.getActiveStatus() != ActiveProxyIp.ActiveStatus.ONLINE) { 168 | return; 169 | } 170 | workThread.post(() -> { 171 | if (activeProxyIp.getActiveStatus() != ActiveProxyIp.ActiveStatus.ONLINE) { 172 | return; 173 | } 174 | DownloadProxyIp remove = cachedProxies.remove(activeProxyIp.getDownloadProxyIp().getResourceId()); 175 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "cache existed: " + (remove != null)); 176 | 177 | removeIfEq(activeProxyIp, ActiveProxyIp::getSeq, poolWithCreateSequence); 178 | removeIfEq(activeProxyIp, ActiveProxyIp::getMurHash, poolWithMurHash); 179 | // removeIfEq(activeProxyIp, ActiveProxyIp::getGeoHashCode, poolWithGeoLocation); 180 | // removeIfEq(activeProxyIp, ActiveProxyIp::getAdminCode, poolWithAdmin); 181 | 182 | activeProxyIp.destroy(dropReason); 183 | 184 | if (poolWithMurHash.size() > runtimeIpSource.getPoolSize().value) { 185 | return; 186 | } 187 | DownloadProxyIp newProxy = detachCacheCachedProxy(); 188 | if (newProxy == null) { 189 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "no cache proxy resource exist, pool size decreased!!"); 190 | return; 191 | } 192 | 193 | runtimeIpSource.recordComposedEvent(userRecorder, () -> "online new proxy: "); 194 | runtimeIpSource.recordComposedMosaicEvent(userRecorder, () -> JSONObject.toJSONString(newProxy)); 195 | 196 | onlineProxyResource(newProxy); 197 | }, true); 198 | // IP下线代码需要立即执行，保证其他任务不会使用到被下线了的ip，特别是ip重试的时候 199 | } 200 | 201 | private void removeIfEq(ActiveProxyIp activeProxyIp, Function kSupplier, Map map) { 202 | T k = kSupplier.apply(activeProxyIp); 203 | if (k == null) { 204 | return; 205 | } 206 | ActiveProxyIp remove = map.remove(k); 207 | if (remove == null) { 208 | return; 209 | } 210 | if (remove != activeProxyIp) { 211 | map.put(k, remove); 212 | } 213 | } 214 | 215 | private DownloadProxyIp detachCacheCachedProxy() { 216 | Iterator> iterator = cachedProxies.entrySet().iterator(); 217 | DownloadProxyIp newProxy = null; 218 | if (iterator.hasNext()) { 219 | Map.Entry firstEntry = iterator.next(); 220 | newProxy = firstEntry.getValue(); 221 | iterator.remove(); 222 | } 223 | return newProxy; 224 | } 225 | 226 | public void offerProxy(DownloadProxyIp downloadProxyIp) { 227 | if (downloadProxyIp.getExpireTime() != null 228 | && downloadProxyIp.getExpireTime() < System.currentTimeMillis() 229 | ) { 230 | // 加入的时候已经过期了 231 | recorder.recordEvent("offer an expired proxy item"); 232 | return; 233 | } 234 | workThread.post(() -> { 235 | if (duplicate(downloadProxyIp)) { 236 | return; 237 | } 238 | 239 | Integer configPoolSize = runtimeIpSource.getPoolSize().value; 240 | if (poolWithMurHash.size() < configPoolSize) { 241 | onlineProxyResource(downloadProxyIp); 242 | return; 243 | } 244 | 245 | cachedProxies.put(downloadProxyIp.getResourceId(), downloadProxyIp); 246 | 247 | // 当缓存的ip资源比较大，超过了ip池的一半，那么我们直接插入替换 248 | // 最早加入的，当前没有为业务服务的资源，并把他替换到二级资源池 249 | if (cachedProxies.size() > Math.max(configPoolSize * 0.5, 1)) { 250 | ActiveProxyIp toReplaceActiveProxyIp = null; 251 | int count = 0; 252 | for (ActiveProxyIp activeProxyIp : poolWithCreateSequence.values()) { 253 | if (activeProxyIp.isIdle()) { 254 | toReplaceActiveProxyIp = activeProxyIp; 255 | break; 256 | } 257 | count++; 258 | if (count > 50) { 259 | break; 260 | } 261 | } 262 | if (toReplaceActiveProxyIp != null) { 263 | recorder.recordEvent("proxy cache overflow , offline old proxy"); 264 | offlineProxy(toReplaceActiveProxyIp, DropReason.IP_IDLE_POOL_OVERFLOW, Recorder.nop); 265 | } 266 | } 267 | }); 268 | } 269 | 270 | private boolean duplicate(DownloadProxyIp downloadProxyIp) { 271 | if (poolWithMurHash.containsKey(ConsistentHashUtil.murHash(downloadProxyIp.getResourceId()))) { 272 | return true; 273 | } 274 | return cachedProxies.containsKey(downloadProxyIp.getResourceId()); 275 | } 276 | 277 | private void onlineProxyResource(DownloadProxyIp downloadProxyIp) { 278 | long lifeTime; 279 | if (downloadProxyIp.getExpireTime() != null) { 280 | lifeTime = downloadProxyIp.getExpireTime() - System.currentTimeMillis(); 281 | } else { 282 | Integer maxAlive = runtimeIpSource.getMaxAlive().value; 283 | if (maxAlive == null) { 284 | maxAlive = 300; 285 | } 286 | lifeTime = maxAlive * 1000L; 287 | } 288 | if (lifeTime <= 0) { 289 | return; 290 | } 291 | 292 | 293 | ActiveProxyIp activeProxyIp = new ActiveProxyIp(this, downloadProxyIp, outboundInc.incrementAndGet()); 294 | 295 | // register in multiple function ip pool 296 | poolWithMurHash.put(activeProxyIp.getMurHash(), activeProxyIp); 297 | poolWithCreateSequence.put(activeProxyIp.getSeq(), activeProxyIp); 298 | 299 | // 该ip配置了有效时间，且代理系统无法感知ip失效， 300 | // 故手动完成ip资源的下线 301 | WeakReference weakReference = new WeakReference<>(activeProxyIp); 302 | workThread.postDelay(() -> { 303 | // 可能已经提前下线了，所以这里用软引用 304 | ActiveProxyIp proxyNotGc = weakReference.get(); 305 | if (proxyNotGc != null) { 306 | offlineProxy(proxyNotGc, DropReason.IP_ALIVE_TIME_REACHED, Recorder.nop); 307 | } 308 | }, lifeTime); 309 | } 310 | 311 | public void makeCache() { 312 | LinkedList outbounds = new LinkedList<>(poolWithMurHash.values()); 313 | for (ActiveProxyIp outbound : outbounds) { 314 | outbound.makeCache(); 315 | } 316 | } 317 | 318 | public void destroy() { 319 | poolWithMurHash.values().forEach(activeProxyIp -> activeProxyIp.destroy(DropReason.IP_RESOURCE_CLOSE)); 320 | } 321 | } 322 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/OutboundOperator.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound; 2 | 3 | import cn.iinti.proxycompose.loop.ValueCallback; 4 | import cn.iinti.proxycompose.utils.NettyThreadPools; 5 | import cn.iinti.proxycompose.proxy.Session; 6 | import io.netty.bootstrap.Bootstrap; 7 | import io.netty.channel.*; 8 | import io.netty.channel.socket.nio.NioSocketChannel; 9 | 10 | public class OutboundOperator { 11 | private static final Bootstrap outboundBootstrap = buildOutboundBootstrap(); 12 | 13 | private static Bootstrap buildOutboundBootstrap() { 14 | return new Bootstrap() 15 | .group(NettyThreadPools.outboundGroup) 16 | .channelFactory(NioSocketChannel::new) 17 | //上游链接在不确定协议的时候，无法确定处理器，我们使用一个插桩替代，在获取链接成功之后我们再手动构造 18 | .handler(new StubHandler()) 19 | // 到代理ip的连接时间，设置短一些，设置为5s，如果不成功那么通过其他代理重试 20 | .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000); 21 | } 22 | 23 | @ChannelHandler.Sharable 24 | public static class StubHandler extends ChannelInboundHandlerAdapter { 25 | @Override 26 | public void channelRegistered(ChannelHandlerContext ctx) { 27 | // just make netty framework happy 28 | ctx.pipeline().remove(this); 29 | } 30 | } 31 | 32 | public static void connectToServer(String host, int port, ValueCallback callback) { 33 | // 这里切换线程池的原因是，connect过程会有socket warming up,实践看来他在有些时候会有一定时间的延迟 34 | outboundBootstrap.config().group().execute(() -> outboundBootstrap.connect( 35 | host, port 36 | ).addListener((ChannelFutureListener) future -> { 37 | if (!future.isSuccess()) { 38 | ValueCallback.failed(callback, future.cause()); 39 | } else { 40 | ValueCallback.success(callback, future.channel()); 41 | } 42 | })); 43 | } 44 | 45 | public static void connectToOutbound( 46 | Session session, long sessionHash, String tag, 47 | ActiveProxyIp.ActivityProxyIpBindObserver observer, 48 | ValueCallback callback) { 49 | session.getProxyServer().getProxyCompose() 50 | .connectToOutbound(session, sessionHash, tag, observer, callback); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/downloader/DownloadProxyIp.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.downloader; 2 | 3 | import cn.iinti.proxycompose.resource.ProxyIp; 4 | import com.alibaba.fastjson.JSONObject; 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | import lombok.experimental.Delegate; 8 | 9 | import javax.annotation.Nullable; 10 | 11 | @Setter 12 | @Getter 13 | public class DownloadProxyIp { 14 | 15 | /** 16 | * 测试耗时 17 | */ 18 | @Nullable 19 | private Long testCost; 20 | 21 | /** 22 | * 加入队列时间 23 | */ 24 | private long enQueueTime; 25 | 26 | 27 | @Delegate 28 | private final ProxyIp proxyIp; 29 | 30 | public DownloadProxyIp(ProxyIp proxyIp) { 31 | this.proxyIp = proxyIp; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return "DownloadProxyIp{" + 37 | "testCost=" + testCost + 38 | ", enQueueTime=" + enQueueTime + 39 | ", proxyIp=" + JSONObject.toJSONString(proxyIp) + 40 | '}'; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/downloader/IpDownloader.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.downloader; 2 | 3 | import cn.iinti.proxycompose.Settings; 4 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 5 | import cn.iinti.proxycompose.proxy.RuntimeIpSource; 6 | import cn.iinti.proxycompose.utils.AsyncHttpInvoker; 7 | import cn.iinti.proxycompose.resource.ProxyIp; 8 | import cn.iinti.proxycompose.loop.Looper; 9 | import cn.iinti.proxycompose.loop.ValueCallback; 10 | import cn.iinti.proxycompose.trace.Recorder; 11 | import cn.iinti.proxycompose.trace.impl.SubscribeRecorders; 12 | import cn.iinti.proxycompose.utils.IpUtils; 13 | import org.apache.commons.lang3.BooleanUtils; 14 | import org.apache.commons.lang3.StringUtils; 15 | import org.asynchttpclient.Realm; 16 | import org.asynchttpclient.proxy.ProxyServer; 17 | import org.asynchttpclient.proxy.ProxyType; 18 | 19 | import java.util.List; 20 | import java.util.function.Consumer; 21 | import java.util.function.Predicate; 22 | import java.util.stream.Collectors; 23 | 24 | public class IpDownloader { 25 | private final RuntimeIpSource runtimeIpSource; 26 | private final Looper workThread; 27 | private volatile boolean isDownloading = false; 28 | private final Recorder recorder; 29 | 30 | 31 | public IpDownloader(RuntimeIpSource runtimeIpSource, Recorder recorder, String sourceKey) { 32 | this.runtimeIpSource = runtimeIpSource; 33 | // downloader里面全是异步，但是可能会调用dns服务，并且可能存在对无效的代理域名进行dns解析， 34 | // 这可能存在些许耗时，所以这里我新建一个线程来处理 35 | this.workThread = new Looper("Downloader-" + sourceKey).startLoop(); 36 | this.recorder = recorder; 37 | } 38 | 39 | private static final String dbName = "GeoLite2-City.mmdb"; 40 | 41 | 42 | public void downloadIp() { 43 | workThread.execute(() -> { 44 | String loadUrl = runtimeIpSource.getLoadURL().value; 45 | if (!isHTTPLink(loadUrl)) { 46 | onDownloadResponse(loadUrl); 47 | return; 48 | } 49 | 50 | if (isDownloading) { 51 | return; 52 | } 53 | isDownloading = true; 54 | 55 | AsyncHttpInvoker.get(loadUrl, runtimeIpSource.getRecorder(), value -> workThread.post(() -> { 56 | isDownloading = false; 57 | 58 | if (!value.isSuccess()) { 59 | recorder.recordEvent(() -> "ip source download failed", value.e); 60 | return; 61 | } 62 | onDownloadResponse(value.v); 63 | })); 64 | }); 65 | 66 | } 67 | 68 | @SuppressWarnings("all") 69 | private static boolean isHTTPLink(String url) { 70 | return StringUtils.startsWithAny(url, "http://", "https://"); 71 | } 72 | 73 | private void onDownloadResponse(String response) { 74 | workThread.execute(() -> { 75 | recorder.recordEvent(() -> "download ip response:\n" + response + "\n"); 76 | List proxyIps = runtimeIpSource.getResourceParser() 77 | .parse(response) 78 | .stream() 79 | .filter(proxyIp -> { 80 | if (!proxyIp.isValid()) { 81 | recorder.recordEvent(() -> "invalid parsed proxyIp"); 82 | return false; 83 | } 84 | return true; 85 | }).collect(Collectors.toList()); 86 | 87 | if (proxyIps.isEmpty()) { 88 | return; 89 | } 90 | // fill password from ip source config 91 | String upUserPassword = runtimeIpSource.getUpstreamAuthPassword().value; 92 | if (StringUtils.isNotBlank(upUserPassword)) { 93 | proxyIps.forEach(proxyIpResourceItem -> { 94 | if (StringUtils.isBlank(proxyIpResourceItem.getPassword())) { 95 | proxyIpResourceItem.setPassword(upUserPassword); 96 | } 97 | }); 98 | } 99 | String upUserName = runtimeIpSource.getUpstreamAuthUser().value; 100 | if (StringUtils.isNotBlank(upUserName)) { 101 | proxyIps.forEach(proxyIpResourceItem -> { 102 | if (StringUtils.isBlank(proxyIpResourceItem.getUserName())) { 103 | proxyIpResourceItem.setUserName(upUserName); 104 | } 105 | }); 106 | } 107 | 108 | 109 | Boolean needTest = runtimeIpSource.getNeedTest().value; 110 | recorder.recordEvent(() -> "this ip source configure test switch: " + needTest); 111 | Consumer action = BooleanUtils.isFalse(needTest) ? 112 | this::offerIpResource : 113 | this::runIpQualityTest; 114 | 115 | // 2024年03约21日，医药魔方： 116 | // 公司降本增效，将malenia服务器压缩到4G内存，同时跑malenia+mysql+python服务，导致物理内存不够 117 | // 最终排查在这里可能并发发出几百个网络情况（1毫秒内），这导致系统极短时间内存快速分配，进而引发oom 118 | // fix此问题使用如下策略：ip下载完成，进行分批延时探测入库，对此网络行为进行削峰填谷，10个ip一批并发多次进行ip质量探测 119 | ////////////////////////////////////////////////////////////////////////////////////////////////////////// 120 | // step = ( interval * 1000 * 0.3) / size 121 | // 步长 = (加载间隔 * 1000毫秒 * 在前30%时间内完成探测) / 本次加载数量 122 | long offerStepInMills = runtimeIpSource.getReloadInterval().value * 300 / proxyIps.size(); 123 | 124 | new IpOfferStep(workThread, offerStepInMills, proxyIps, action).execute(); 125 | }); 126 | } 127 | 128 | 129 | private void offerIpResource(DownloadProxyIp downloadProxyIp) { 130 | // 请注意，这里必须确保线程正确，因为InetAddress的解析可能比较耗时 131 | workThread.execute(() -> { 132 | recorder.recordEvent(() -> "prepare enpool proxy ip: " + downloadProxyIp); 133 | runtimeIpSource.getIpPool().offerProxy(downloadProxyIp); 134 | }); 135 | 136 | } 137 | 138 | private void runIpQualityTest(DownloadProxyIp downloadProxyIp) { 139 | recorder.recordEvent(() -> "[QualityTest] begin test proxy quality: " + downloadProxyIp); 140 | Recorder recorderForTester = SubscribeRecorders.IP_TEST.acquireRecorder( 141 | downloadProxyIp.getResourceId() + "_" + System.currentTimeMillis(), 142 | Settings.global.debug.value, runtimeIpSource.getName() 143 | ); 144 | recorderForTester.recordEvent(() -> "[QualityTest] begin to test proxy:" + downloadProxyIp); 145 | 146 | String url = Settings.global.proxyHttpTestURL.value; 147 | recorder.recordEvent(() -> "[QualityTest] ip test with url: " + url); 148 | long startTestTimestamp = System.currentTimeMillis(); 149 | 150 | ProxyServer.Builder proxyBuilder = new ProxyServer.Builder(downloadProxyIp.getProxyHost(), downloadProxyIp.getProxyPort()); 151 | 152 | if (runtimeIpSource.needAuth()) { 153 | proxyBuilder.setRealm(new Realm.Builder( 154 | runtimeIpSource.getUpstreamAuthUser().value, 155 | runtimeIpSource.getUpstreamAuthPassword().value 156 | ).setScheme(Realm.AuthScheme.BASIC)); 157 | } 158 | 159 | if (runtimeIpSource.getSupportProtocolList().stream().anyMatch(Predicate.isEqual(Protocol.SOCKS5))) { 160 | // 有代理资源只能支持socks5，所以如果代理支持socks5时，直接使用s5来代理， 161 | // 默认将会使用http协议族，然后malenia具备自动的协议转换能力 162 | recorder.recordEvent(() -> "[QualityTest] use test protocol SOCKS_V5"); 163 | proxyBuilder.setProxyType(ProxyType.SOCKS_V5); 164 | } else { 165 | recorder.recordEvent(() -> "[QualityTest] use test protocol HTTP"); 166 | } 167 | 168 | AsyncHttpInvoker.get(url, recorderForTester, proxyBuilder, value -> { 169 | if (value.isSuccess()) { 170 | if (!IpUtils.isValidIp(value.v)) { 171 | // 扭转成功状态，因为响应的内容不是ip，那么认为报文错误 172 | value = ValueCallback.Value.failed("response not ip format: " + value.v); 173 | } 174 | } 175 | 176 | if (!value.isSuccess()) { 177 | recorder.recordEvent(() -> "[QualityTest] ip test failed", value.e); 178 | recorderForTester.recordEvent(() -> "ip test failed", value.e); 179 | return; 180 | } 181 | recorder.recordEvent(() -> "[QualityTest] ip test success"); 182 | downloadProxyIp.setOutIp(value.v); 183 | downloadProxyIp.setTestCost(System.currentTimeMillis() - startTestTimestamp); 184 | offerIpResource(downloadProxyIp); 185 | }); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/downloader/IpOfferStep.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.downloader; 2 | 3 | import cn.iinti.proxycompose.resource.ProxyIp; 4 | import cn.iinti.proxycompose.loop.Looper; 5 | 6 | import java.util.List; 7 | import java.util.function.Consumer; 8 | 9 | public class IpOfferStep { 10 | private static final int offerBatch = 10; 11 | private int nowIndex; 12 | private final Looper workThread; 13 | private final long offerStepInMills; 14 | private final List downloadIps; 15 | private final Consumer offerAction; 16 | 17 | public IpOfferStep(Looper workThread, long offerStepInMills, List downloadIps, 18 | Consumer offerAction) { 19 | this.workThread = workThread; 20 | this.offerStepInMills = offerStepInMills; 21 | this.downloadIps = downloadIps; 22 | this.offerAction = offerAction; 23 | this.nowIndex = 0; 24 | } 25 | 26 | void execute() { 27 | int maxIndex = downloadIps.size() - 1; 28 | for (int i = 0; i < offerBatch; i++) { 29 | ProxyIp proxyIp = downloadIps.get(this.nowIndex).resolveId(); 30 | offerAction.accept(new DownloadProxyIp(proxyIp)); 31 | this.nowIndex++; 32 | if (this.nowIndex > maxIndex) { 33 | return; 34 | } 35 | } 36 | workThread.postDelay(this::execute, offerStepInMills * offerBatch); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/handshark/AbstractUpstreamHandShaker.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.handshark; 2 | 3 | 4 | import cn.iinti.proxycompose.Settings; 5 | import cn.iinti.proxycompose.loop.ValueCallback; 6 | import cn.iinti.proxycompose.proxy.Session; 7 | import cn.iinti.proxycompose.proxy.outbound.ActiveProxyIp; 8 | import cn.iinti.proxycompose.proxy.outbound.IpPool; 9 | import cn.iinti.proxycompose.trace.impl.SubscribeRecorders; 10 | import io.netty.channel.Channel; 11 | 12 | import java.lang.ref.WeakReference; 13 | import java.util.HashSet; 14 | import java.util.Set; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.concurrent.atomic.AtomicBoolean; 17 | 18 | public abstract class AbstractUpstreamHandShaker { 19 | protected Session session; 20 | protected Channel outboundChannel; 21 | private final ValueCallback callback; 22 | protected final ActiveProxyIp activeProxyIp; 23 | 24 | protected SubscribeRecorders.SubscribeRecorder recorder; 25 | private final AtomicBoolean hasEmitResult = new AtomicBoolean(false); 26 | 27 | /** 28 | * @param session 对应隧道 29 | * @param callback 回掉函数，请注意他包裹一个Boolean对象，表示是否可以failover 30 | */ 31 | public AbstractUpstreamHandShaker(Session session, Channel outboundChannel, ActiveProxyIp activeProxyIp, ValueCallback callback) { 32 | this.session = session; 33 | this.activeProxyIp = activeProxyIp; 34 | this.callback = value -> { 35 | if (!value.isSuccess()) { 36 | // 握手失败的链接，统一计数为失败 37 | outboundChannel.close(); 38 | } 39 | callback.onReceiveValue(value); 40 | }; 41 | this.recorder = session.getRecorder(); 42 | this.outboundChannel = outboundChannel; 43 | setupHandSharkTimeout(); 44 | 45 | } 46 | 47 | private void setupHandSharkTimeout() { 48 | // 看追踪日志，部分请求出现在发送了connect之后没有回应，所以这里我们做一次检测 49 | // 设定一定timeout之后，主动阻断请求，然后执行failover 50 | // 猜想出现这个的愿意如下： 51 | // 1. 代理服务器bug，导致代理服务器接受了connect但是不给响应 52 | // 2. 代理服务器在我们发送connect之后发生了重播，或者发生了线路无法通畅连接的问题 53 | // 3. 代理服务器到真实服务器的线路bug，或代理服务器本身代理socket创建过程timeout过长，或者代理服务器连接真实服务器失败了，但是代理服务器没有较好的处理这个问题 54 | // 55 | // 实际测试发现这个值普遍在2秒左右，我们默认值设置为5s 56 | // 请注意，这个逻辑是必要的，这是因为如果我们不设置超时。上游ip资源不关闭链接，将会导致链接泄漏，影响GC同时消耗FD资源 57 | // 第一代的malenia系统socks代理缺失了time-out检查，导致出现了200w FD资源无法回收的情况 58 | Integer connectTimeout = Settings.global.handleSharkConnectionTimeout.value; 59 | // min:1s max: 60s default: 5s 60 | if (connectTimeout == null || connectTimeout > 60 * 1000) { 61 | connectTimeout = 5000; 62 | } 63 | if (connectTimeout < 1000) { 64 | connectTimeout = 1000; 65 | } 66 | recorder.recordEvent("this connection timeout is: " + connectTimeout); 67 | WeakReference ref = new WeakReference<>(this); 68 | outboundChannel.eventLoop().schedule(() -> { 69 | AbstractUpstreamHandShaker upstreamHandShaker = ref.get(); 70 | if (upstreamHandShaker == null) { 71 | return; 72 | } 73 | if (upstreamHandShaker.hasEmitResult.get()) { 74 | return; 75 | } 76 | upstreamHandShaker.recorder.recordEvent(() -> "timeout"); 77 | upstreamHandShaker.emitFailed("upstream hand shark timeout, abort this connection", true); 78 | }, connectTimeout, TimeUnit.MILLISECONDS); 79 | 80 | } 81 | 82 | protected void emitSuccess() { 83 | if (hasEmitResult.compareAndSet(false, true)) { 84 | ValueCallback.success(callback, true); 85 | } 86 | } 87 | 88 | private static final Set notOfflineProxyMsgs = new HashSet() { 89 | { 90 | add("NETWORK_UNREACHABLE"); 91 | } 92 | }; 93 | 94 | protected void emitFailed(Object msg, boolean canRetry) { 95 | if (hasEmitResult.compareAndSet(false, true)) { 96 | ValueCallback.Value value; 97 | if (msg instanceof Throwable) { 98 | value = ValueCallback.Value.failed((Throwable) msg); 99 | } else { 100 | value = ValueCallback.Value.failed(msg.toString()); 101 | } 102 | value.v = canRetry; 103 | String errorMsg = value.e.getMessage(); 104 | recorder.recordEvent(() -> "HandShark failed", value.e); 105 | try { 106 | callback.onReceiveValue(value); 107 | } finally { 108 | boolean needOfflineProxy = true; 109 | for (String notOfflineProxyMsg : notOfflineProxyMsgs) { 110 | if (errorMsg.contains(notOfflineProxyMsg)) { 111 | // java.lang.RuntimeException: cmd failed: NETWORK_UNREACHABLE 112 | // 这个时候其实就是目标服务器连不上，和代理服务器没有关系 113 | needOfflineProxy = false; 114 | break; 115 | } 116 | } 117 | if (needOfflineProxy) { 118 | // 代理云ip过期之后，可能允许连接，但是鉴权失败，这是我们建议建议下线ip 119 | ActiveProxyIp.offlineBindingProxy( 120 | outboundChannel, 121 | IpPool.OfflineLevel.SUGGEST, 122 | recorder); 123 | } 124 | } 125 | 126 | 127 | } 128 | } 129 | 130 | 131 | public abstract void doHandShark(); 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/handshark/HandShakerHttps.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.handshark; 2 | 3 | import cn.iinti.proxycompose.loop.ValueCallback; 4 | import cn.iinti.proxycompose.proxy.Session; 5 | import cn.iinti.proxycompose.proxy.outbound.ActiveProxyIp; 6 | import io.netty.channel.Channel; 7 | import io.netty.channel.ChannelHandlerContext; 8 | import io.netty.channel.ChannelPipeline; 9 | import io.netty.channel.SimpleChannelInboundHandler; 10 | import io.netty.handler.codec.http.*; 11 | 12 | /** 13 | * 完成https和上级代理的鉴权和握手 14 | */ 15 | public class HandShakerHttps extends AbstractUpstreamHandShaker { 16 | private boolean hasSendAuthentication = false; 17 | 18 | 19 | /** 20 | * @param session 对应隧道 21 | * @param callback 回掉函数，请注意他包裹一个Boolean对象，表示是否可以failover 22 | */ 23 | public HandShakerHttps(Session session, Channel outboundChannel, ActiveProxyIp activeProxyIp, ValueCallback callback) { 24 | super(session, outboundChannel, activeProxyIp, callback); 25 | } 26 | 27 | @Override 28 | public void doHandShark() { 29 | recorder.recordEvent(() -> "begin https upstream hand shark"); 30 | 31 | ChannelPipeline pipeline = outboundChannel.pipeline(); 32 | pipeline.addFirst(new HttpRequestEncoder()); 33 | pipeline.addFirst(new HttpResponseDecoder()); 34 | 35 | DefaultFullHttpRequest connectRequest = createConnectRequest(); 36 | outboundChannel.writeAndFlush(connectRequest) 37 | .addListener(future -> { 38 | if (!future.isSuccess()) { 39 | recorder.recordEvent(() -> "write https connect request failed"); 40 | emitFailed("write https connect request failed", true); 41 | return; 42 | } 43 | recorder.recordEvent(() -> "request write finish ,setupResponseHandler"); 44 | outboundChannel.pipeline().addLast(new SimpleChannelInboundHandler() { 45 | @Override 46 | protected void channelRead0(ChannelHandlerContext ctx, HttpResponse msg) { 47 | handleResponse(msg, this); 48 | } 49 | 50 | @Override 51 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 52 | // 发connect报文过程发现对方关闭了链接，这个时候可以重试 53 | recorder.recordEvent(() -> "exception: ", cause); 54 | emitFailed(cause, true); 55 | } 56 | }); 57 | }); 58 | 59 | } 60 | 61 | 62 | private void handleResponse(HttpResponse httpResponse, SimpleChannelInboundHandler mHandler) { 63 | int code = httpResponse.status().code(); 64 | recorder.recordEvent(() -> "ConnectHttpResponse:" + code); 65 | if (code == HttpResponseStatus.OK.code()) { 66 | recorder.recordEvent(() -> "hand shark success, remove upstream http netty handler"); 67 | ChannelPipeline pipeline = outboundChannel.pipeline(); 68 | pipeline.remove(HttpRequestEncoder.class); 69 | pipeline.remove(HttpResponseDecoder.class); 70 | pipeline.remove(mHandler); 71 | emitSuccess(); 72 | return; 73 | } 74 | 75 | if (code >= 500) { 76 | recorder.recordEvent(() -> "receive 5xx from upstream" + httpResponse); 77 | emitFailed(httpResponse.status().reasonPhrase(), true); 78 | return; 79 | } 80 | 81 | if (code == 401) { 82 | // 401 芝麻代理 节点失效错误,这是在瞎搞 83 | recorder.recordEvent(() -> "401 on hand shark"); 84 | emitFailed(httpResponse.status().reasonPhrase(), true); 85 | return; 86 | } 87 | if (code != HttpResponseStatus.PROXY_AUTHENTICATION_REQUIRED.code()) { 88 | recorder.recordEvent(() -> "not 407 response,hand shark failed"); 89 | emitFailed(httpResponse.status().reasonPhrase(), false); 90 | return; 91 | } 92 | 93 | if (hasSendAuthentication) { 94 | recorder.recordEvent(() -> "upstream password error"); 95 | emitFailed("upstream password error", false); 96 | return; 97 | } 98 | 99 | // 有部分代理服务器首次发送了代理鉴权内容但是他不认，依然返回407 100 | // 然后有一些网络库一旦首次发送过密码，即使407也不会重新发送带鉴权请求，而是直接报告失败 101 | // 所以我们这里屏蔽这个问题，如果407我们再发送一次报文. 102 | recorder.recordEvent(() -> "407 response, send user pass again"); 103 | DefaultFullHttpRequest connectRequest = createConnectRequest(); 104 | 105 | outboundChannel.writeAndFlush(connectRequest) 106 | .addListener(future -> { 107 | if (!future.isSuccess()) { 108 | recorder.recordEvent(() -> "write https connect request failed"); 109 | emitFailed("write https connect request failed", true); 110 | return; 111 | } 112 | recorder.recordEvent(() -> "second auth request write finish"); 113 | hasSendAuthentication = true; 114 | }); 115 | } 116 | 117 | 118 | private DefaultFullHttpRequest createConnectRequest() { 119 | DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, 120 | HttpMethod.CONNECT, session.getConnectTarget().getIpPort()); 121 | 122 | String header = activeProxyIp.buildHttpAuthenticationInfo(); 123 | 124 | if (header == null) { 125 | recorder.recordEvent(() -> "this ipSource do not need authentication"); 126 | } else { 127 | request.headers().add(HttpHeaderNames.PROXY_AUTHORIZATION, header); 128 | recorder.recordMosaicMsg(() -> "fill authorizationContent: " + header); 129 | } 130 | return request; 131 | } 132 | 133 | 134 | } 135 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/handshark/HandShakerSocks5.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.handshark; 2 | 3 | import cn.iinti.proxycompose.resource.IpAndPort; 4 | import cn.iinti.proxycompose.proxy.Session; 5 | import cn.iinti.proxycompose.proxy.outbound.ActiveProxyIp; 6 | import cn.iinti.proxycompose.utils.IpUtils; 7 | import cn.iinti.proxycompose.loop.ValueCallback; 8 | import com.google.common.collect.Lists; 9 | import io.netty.channel.Channel; 10 | import io.netty.channel.ChannelHandlerContext; 11 | import io.netty.channel.ChannelPipeline; 12 | import io.netty.channel.SimpleChannelInboundHandler; 13 | import io.netty.handler.codec.socks.*; 14 | import io.netty.util.ReferenceCountUtil; 15 | 16 | import java.util.List; 17 | 18 | public class HandShakerSocks5 extends AbstractUpstreamHandShaker { 19 | private final ChannelPipeline pipeline; 20 | 21 | /** 22 | * @param turning 对应隧道 23 | * @param callback 回掉函数，请注意他包裹一个Boolean对象，表示是否可以failover 24 | */ 25 | public HandShakerSocks5(Session turning, Channel outboundChannel, ActiveProxyIp activeProxyIp, ValueCallback callback) { 26 | super(turning, outboundChannel, activeProxyIp, callback); 27 | pipeline = outboundChannel.pipeline(); 28 | } 29 | 30 | @Override 31 | public void doHandShark() { 32 | recorder.recordEvent(() -> "begin socks5 hand shark"); 33 | 34 | pipeline.addLast(new SocksMessageEncoder()); 35 | List socksAuthSchemes = Lists.newLinkedList(); 36 | socksAuthSchemes.add(SocksAuthScheme.NO_AUTH); 37 | if (activeProxyIp.canUserPassAuth()) { 38 | socksAuthSchemes.add(SocksAuthScheme.AUTH_PASSWORD); 39 | } 40 | SocksInitRequest socksInitRequest = new SocksInitRequest(socksAuthSchemes); 41 | recorder.recordEvent(() -> "write socksInitRequest"); 42 | 43 | outboundChannel.writeAndFlush(socksInitRequest) 44 | .addListener(future -> { 45 | if (!future.isSuccess()) { 46 | recorder.recordEvent(() -> "write socksInitRequest failed"); 47 | emitFailed(future.cause(), true); 48 | return; 49 | } 50 | recorder.recordEvent(() -> "setup socks InitResponse handler"); 51 | pipeline.addFirst(new SocksInitResponseDecoder()); 52 | pipeline.addLast(new SocksResponseHandler()); 53 | }); 54 | } 55 | 56 | 57 | private void doConnectToUpstream() { 58 | // use proxy switch ,this means we just need support ipv4 & tcp 59 | IpAndPort ipAndPort = session.getConnectTarget(); 60 | 61 | SocksCmdRequest socksCmdRequest = new SocksCmdRequest(SocksCmdType.CONNECT, 62 | IpUtils.isIpV4(ipAndPort.getIp()) ? SocksAddressType.IPv4 : SocksAddressType.DOMAIN, 63 | ipAndPort.getIp(), ipAndPort.getPort()); 64 | 65 | recorder.recordEvent(() -> "send cmd request to upstream"); 66 | pipeline.addFirst(new SocksCmdResponseDecoder()); 67 | outboundChannel.writeAndFlush(socksCmdRequest) 68 | .addListener(future -> { 69 | if (!future.isSuccess()) { 70 | recorder.recordEvent(() -> "write upstream socksCmdRequest failed"); 71 | emitFailed(future.cause(), true); 72 | } else { 73 | recorder.recordEvent(() -> "write upstream socksCmdRequest finished"); 74 | // now waiting cmd response 75 | } 76 | }); 77 | } 78 | 79 | private void doAuth() { 80 | if (!activeProxyIp.canUserPassAuth()) { 81 | emitFailed("the upstream server need auth,but no auth configured for this ip source : " 82 | + activeProxyIp.getIpPool().getRuntimeIpSource().getName(), false); 83 | return; 84 | } 85 | recorder.recordMosaicMsg(() -> "send socks5 auth"); 86 | pipeline.addFirst(new SocksAuthResponseDecoder()); 87 | SocksAuthRequest socksAuthRequest = new SocksAuthRequest( 88 | activeProxyIp.getDownloadProxyIp().getUserName(), 89 | activeProxyIp.getDownloadProxyIp().getPassword() 90 | ); 91 | outboundChannel.writeAndFlush(socksAuthRequest) 92 | .addListener(future -> { 93 | if (!future.isSuccess()) { 94 | emitFailed(future.cause(), true); 95 | return; 96 | } 97 | recorder.recordEvent(() -> "auth request send finish"); 98 | }); 99 | } 100 | 101 | private void handleInitResponse(SocksInitResponse response) { 102 | SocksAuthScheme socksAuthScheme = response.authScheme(); 103 | switch (socksAuthScheme) { 104 | case NO_AUTH: 105 | doConnectToUpstream(); 106 | break; 107 | case AUTH_PASSWORD: 108 | doAuth(); 109 | break; 110 | default: 111 | emitFailed("no support auth method: " + socksAuthScheme, false); 112 | } 113 | } 114 | 115 | private void handleAuthResponse(SocksAuthResponse response) { 116 | recorder.recordEvent(() -> "socks5 handShark authResponse: " + response.authStatus()); 117 | if (response.authStatus() != SocksAuthStatus.SUCCESS) { 118 | emitFailed("upstream auth failed", false); 119 | return; 120 | } 121 | doConnectToUpstream(); 122 | } 123 | 124 | private void handleCMDResponse(SocksCmdResponse response) { 125 | recorder.recordEvent(() -> "upstream CMD Response: " + response); 126 | if (response.cmdStatus() != SocksCmdStatus.SUCCESS) { 127 | recorder.recordEvent(() -> "cmd failed: " + response.cmdStatus()); 128 | emitFailed("cmd failed: " + response.cmdStatus(), true); 129 | return; 130 | } 131 | pipeline.remove(SocksMessageEncoder.class); 132 | pipeline.remove(SocksResponseHandler.class); 133 | recorder.recordEvent(() -> "upstream HandShark success finally"); 134 | emitSuccess(); 135 | } 136 | 137 | private void handleUpstreamSocksResponse(SocksResponse msg) { 138 | switch (msg.responseType()) { 139 | case INIT: 140 | handleInitResponse((SocksInitResponse) msg); 141 | break; 142 | case AUTH: 143 | handleAuthResponse((SocksAuthResponse) msg); 144 | break; 145 | case CMD: 146 | handleCMDResponse((SocksCmdResponse) msg); 147 | break; 148 | default: 149 | recorder.recordEvent(() -> "unknown socksResponse: " + msg); 150 | emitFailed("unknown socksResponse:" + msg, false); 151 | } 152 | } 153 | 154 | private class SocksResponseHandler extends SimpleChannelInboundHandler { 155 | 156 | @Override 157 | protected void channelRead0(ChannelHandlerContext ctx, SocksResponse msg) throws Exception { 158 | handleUpstreamSocksResponse(msg); 159 | } 160 | 161 | @Override 162 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { 163 | try { 164 | emitFailed(cause, true); 165 | } finally { 166 | ReferenceCountUtil.release(cause); 167 | } 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/handshark/Protocol.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.handshark; 2 | 3 | import lombok.Getter; 4 | 5 | public enum Protocol { 6 | HTTP(0), 7 | // 这里的https实际上指代connect，实践中都是作为https使用。但是实际上也是可以跑http为加密流量的 8 | HTTPS(1), 9 | SOCKS5(2); 10 | 11 | static { 12 | HTTP.supportOverlayProtocol = new Protocol[]{}; 13 | // 理论上https也是可以支持转发到socks上面的，但是他只能转发socks的tcp部分， 14 | // 为了避免未来支持udp的时候出现问题，我们不配置socks流量跑到https信道上面来 15 | HTTPS.supportOverlayProtocol = new Protocol[]{HTTP}; 16 | // socks4和sock5可以相互转换，如有有上游供应商只支持socks4，我们也可以提供全套的代理服务 17 | SOCKS5.supportOverlayProtocol = new Protocol[]{HTTP, HTTPS}; 18 | } 19 | 20 | @Getter 21 | private Protocol[] supportOverlayProtocol; 22 | 23 | @Getter 24 | private final int priority; 25 | 26 | Protocol(int priority) { 27 | this.priority = priority; 28 | } 29 | 30 | public static Protocol get(String name) { 31 | for (Protocol protocol : Protocol.values()) { 32 | if (protocol.name().equalsIgnoreCase(name)) { 33 | return protocol; 34 | } 35 | } 36 | return null; 37 | } 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/outbound/handshark/ProtocolManager.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.outbound.handshark; 2 | 3 | import cn.iinti.proxycompose.loop.ValueCallback; 4 | import cn.iinti.proxycompose.proxy.Session; 5 | import cn.iinti.proxycompose.proxy.outbound.ActiveProxyIp; 6 | import io.netty.channel.Channel; 7 | 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | public class ProtocolManager { 13 | 14 | private static final Map upstreamHandShakerMap = new HashMap() { 15 | { 16 | put(Protocol.HTTP, (turning, outboundChannel, activeProxyIp, callback) -> 17 | new AbstractUpstreamHandShaker(turning, outboundChannel, activeProxyIp, callback) { 18 | @Override 19 | public void doHandShark() { 20 | // 请注意，不允许直接在这里操作callback，因为handshake需要统一管理他 21 | emitSuccess(); 22 | } 23 | }); 24 | put(Protocol.HTTPS, HandShakerHttps::new); 25 | put(Protocol.SOCKS5, HandShakerSocks5::new); 26 | } 27 | }; 28 | 29 | private interface UpstreamHandShakerFactory { 30 | AbstractUpstreamHandShaker create(Session session, Channel outboundChannel, ActiveProxyIp activeProxyIp, ValueCallback callback); 31 | } 32 | 33 | public static AbstractUpstreamHandShaker createHandShaker(Protocol protocol, Session turning, Channel outboundChannel, 34 | ActiveProxyIp activeProxyIp, ValueCallback callback) { 35 | if (protocol != null) { 36 | UpstreamHandShakerFactory factory = upstreamHandShakerMap.get(protocol); 37 | if (factory != null) { 38 | return factory.create(turning, outboundChannel, activeProxyIp, callback); 39 | } 40 | } 41 | return new AbstractUpstreamHandShaker(turning, outboundChannel, activeProxyIp, callback) { 42 | @Override 43 | public void doHandShark() { 44 | ValueCallback.failed(callback, "unknown protocol: " + protocol); 45 | } 46 | }; 47 | } 48 | 49 | 50 | public static Protocol chooseUpstreamProtocol(Protocol inboundProtocol, List supportList) { 51 | if (inboundProtocol == Protocol.HTTP) { 52 | // 特殊逻辑，如果是http，我们不让上游代理走http，而是尽量走socks 53 | // 在代理失败判定过程，纯http的判定需要侵入到http协议报文中感知，并且可能因为上游代理服务器的实现导致存在可能的误判 54 | // 这是因为http的鉴权、连接建立、业务请求发送是来自同一个请求流程。我们可以考虑实现http侵入感知，但是这会带来巨大的系统开销和编程难度 55 | // 并且在上游代理系统不标准实现的情况下，我们可能存在对代理质量的误判 56 | for (Protocol candidateProtocol : supportList) { 57 | if (candidateProtocol == Protocol.HTTP) { 58 | continue; 59 | } 60 | for (Protocol protocol : candidateProtocol.getSupportOverlayProtocol()) { 61 | if (protocol == inboundProtocol) { 62 | return candidateProtocol; 63 | } 64 | } 65 | } 66 | } 67 | 68 | for (Protocol support : supportList) { 69 | // 首先使用原生的上下游协议 70 | if (support == inboundProtocol) { 71 | return support; 72 | } 73 | } 74 | 75 | for (Protocol candidateProtocol : supportList) { 76 | for (Protocol protocol : candidateProtocol.getSupportOverlayProtocol()) { 77 | if (protocol == inboundProtocol) { 78 | return candidateProtocol; 79 | } 80 | } 81 | } 82 | return null; 83 | } 84 | 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/switcher/EatErrorFilter.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.switcher; 2 | 3 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 4 | import cn.iinti.proxycompose.trace.Recorder; 5 | import io.netty.channel.Channel; 6 | 7 | public class EatErrorFilter implements UpstreamHandSharkCallback { 8 | private final UpstreamHandSharkCallback delegate; 9 | private final Recorder recorder; 10 | 11 | public EatErrorFilter(UpstreamHandSharkCallback delegate, Recorder recorder) { 12 | this.delegate = delegate; 13 | this.recorder = recorder; 14 | } 15 | 16 | private volatile boolean hasSendErrorMsg = false; 17 | 18 | @Override 19 | public void onHandSharkFinished(Channel upstreamChannel, Protocol outboundProtocol) { 20 | if (hasSendErrorMsg) { 21 | recorder.recordEvent(() -> "hasSendErrorMsg ignore callback onHandSharkFinished"); 22 | return; 23 | } 24 | delegate.onHandSharkFinished(upstreamChannel,outboundProtocol); 25 | } 26 | 27 | @Override 28 | public void onHandSharkError(Throwable e) { 29 | if (hasSendErrorMsg) { 30 | recorder.recordEvent(() -> "duplicate callback on hand shark"); 31 | return; 32 | } 33 | hasSendErrorMsg = true; 34 | delegate.onHandSharkError(e); 35 | } 36 | } -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/switcher/OutboundConnectTask.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.switcher; 2 | 3 | 4 | import cn.iinti.proxycompose.Settings; 5 | import cn.iinti.proxycompose.proxy.ProxyCompose; 6 | import cn.iinti.proxycompose.proxy.Session; 7 | import cn.iinti.proxycompose.proxy.outbound.ActiveProxyIp; 8 | import cn.iinti.proxycompose.proxy.outbound.OutboundOperator; 9 | import cn.iinti.proxycompose.proxy.outbound.handshark.AbstractUpstreamHandShaker; 10 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 11 | import cn.iinti.proxycompose.proxy.outbound.handshark.ProtocolManager; 12 | import cn.iinti.proxycompose.utils.ConsistentHashUtil; 13 | import cn.iinti.proxycompose.loop.ParallelExecutor; 14 | import cn.iinti.proxycompose.loop.ValueCallback; 15 | import cn.iinti.proxycompose.trace.Recorder; 16 | import io.netty.channel.Channel; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * 这是本系统最核心的心脏部位，他决定了整个ip池的上下游连接选择过程 22 | */ 23 | public class OutboundConnectTask implements ActiveProxyIp.ActivityProxyIpBindObserver { 24 | private final Session session; 25 | private final UpstreamHandSharkCallback callback; 26 | private final Recorder recorder; 27 | // 失败重试次数 28 | public int failoverCount = 0; 29 | private String resolvedIpSource = "unknown"; 30 | 31 | public static void startConnectOutbound(Session session, UpstreamHandSharkCallback callback) { 32 | new OutboundConnectTask(session, callback).doStart(); 33 | } 34 | 35 | private OutboundConnectTask(Session session, UpstreamHandSharkCallback callback) { 36 | this.session = session; 37 | this.recorder = session.getRecorder(); 38 | this.callback = new EatErrorFilter(callback, recorder); 39 | } 40 | 41 | private void doStart() { 42 | proxyForward(null); 43 | } 44 | 45 | 46 | private void proxyForward(Throwable throwable) { 47 | if (failoverCount >= Settings.global.maxFailoverCount.value) { 48 | recorder.recordEvent(() -> "failoverCount count: " + failoverCount + " maxCount:" + Settings.global.maxFailoverCount.value); 49 | if (throwable == null) { 50 | throwable = new RuntimeException("get upstream failed"); 51 | } 52 | callback.onHandSharkError(throwable); 53 | return; 54 | } 55 | failoverCount++; 56 | 57 | ValueCallback channelCallback = makeChannelFinishedEvent(); 58 | 59 | long sessionHash = session.getSessionHash(); 60 | ProxyCompose proxyCompose = session.getProxyServer().getProxyCompose(); 61 | 62 | if (failoverCount == 1) { 63 | // 如果是第一次尝试创建连接，则使用单并发，并且尝试记录曾经的隧道规则 64 | // 尽可能保证隧道映射不变 65 | proxyCompose.fetchCachedSession(session, value -> { 66 | if (value.isSuccess()) { 67 | recorder.recordEvent(() -> "this request session hold, reuse cached ip"); 68 | value.v.borrowConnect(recorder, "CachedIp", this, channelCallback); 69 | } else { 70 | recorder.recordEvent(() -> "fist choose ip resource"); 71 | OutboundOperator.connectToOutbound(session, sessionHash, "first create-> ", this, channelCallback); 72 | } 73 | }); 74 | return; 75 | } 76 | // 否则使用并发的方式，连续创建三个连接，并且记忆成功的隧道 77 | recorder.recordEvent(() -> "retry index: " + failoverCount); 78 | failoverConnect(channelCallback, sessionHash, proxyCompose); 79 | } 80 | 81 | @Override 82 | public void onBind(ActiveProxyIp activeProxyIp) { 83 | resolvedIpSource = activeProxyIp.getIpPool() 84 | .getRuntimeIpSource().getName(); 85 | } 86 | 87 | 88 | private static class FailoverTaskHolder { 89 | final long hash; 90 | final Channel channel; 91 | final int index; 92 | final String tag; 93 | 94 | public FailoverTaskHolder(long hash, int index, Channel value, String tag) { 95 | this.hash = hash; 96 | this.index = index; 97 | this.channel = value; 98 | this.tag = tag; 99 | } 100 | } 101 | 102 | 103 | private class FailoverMsgEvent implements ParallelExecutor.ParallelConnectEvent { 104 | private final ValueCallback channelCallback; 105 | 106 | public FailoverMsgEvent(ValueCallback channelCallback) { 107 | this.channelCallback = channelCallback; 108 | } 109 | 110 | @Override 111 | public void firstSuccess(ValueCallback.Value value) { 112 | FailoverTaskHolder holder = value.v; 113 | recorder.recordEvent(holder.tag + "create connection success :"); 114 | recorder.recordMosaicMsgIfSubscribeRecorder(() -> holder.tag + holder.channel); 115 | ValueCallback.success(channelCallback, holder.channel); 116 | } 117 | 118 | @Override 119 | public void secondSuccess(ValueCallback.Value value) { 120 | FailoverTaskHolder holder = value.v; 121 | recorder.recordEvent(() -> holder.tag + "create channel secondSuccess, restore cache channel"); 122 | ActiveProxyIp.restoreCache(holder.channel); 123 | } 124 | 125 | @Override 126 | public void finalFailed(Throwable throwable) { 127 | recorder.recordEvent(() -> "all ip create connection failed", throwable); 128 | ValueCallback.failed(channelCallback, throwable); 129 | } 130 | } 131 | 132 | private static final int parallelSize = 3; 133 | 134 | private void failoverConnect(ValueCallback channelCallback, long sessionHash, ProxyCompose proxyCompose) { 135 | // 不是第一次，那么同时使用多个资源创建ip，谁成功以谁为准 136 | ParallelExecutor executor = new ParallelExecutor<>(proxyCompose.getComposeWorkThead(), parallelSize, 137 | new FailoverMsgEvent(channelCallback)); 138 | 139 | for (int i = 0; i < parallelSize; i++) { 140 | long newHash = ConsistentHashUtil.murHash(String.valueOf(sessionHash) + i + failoverCount); 141 | String tag = Math.abs(newHash) + "_" + i + " -> "; 142 | recorder.recordEvent(() -> tag + "parallel create connection: "); 143 | int index = i; 144 | OutboundOperator.connectToOutbound(session, newHash, tag, this, 145 | value -> { 146 | if (value.isSuccess()) { 147 | executor.onReceiveValue( 148 | ValueCallback.Value.success(new FailoverTaskHolder(newHash, index, value.v, tag)) 149 | ); 150 | } else { 151 | executor.onReceiveValue(value.errorTransfer()); 152 | } 153 | } 154 | ); 155 | } 156 | } 157 | 158 | private ValueCallback makeChannelFinishedEvent() { 159 | return channelValue -> { 160 | if (!channelValue.isSuccess()) { 161 | // retry 162 | proxyForward(channelValue.e); 163 | return; 164 | } 165 | Channel channel = channelValue.v; 166 | 167 | // 成功之后，和上游ip进行鉴权 168 | ActiveProxyIp activeProxyIp = ActiveProxyIp.getBinding(channel); 169 | List supportProtocolList = activeProxyIp.getIpPool().getRuntimeIpSource().getSupportProtocolList(); 170 | 171 | Protocol outboundProtocol = ProtocolManager.chooseUpstreamProtocol(session.getInboundProtocol(), 172 | supportProtocolList); 173 | 174 | if (outboundProtocol == null) { 175 | callback.onHandSharkError(new RuntimeException("no support protocol from upstream")); 176 | channel.close(); 177 | return; 178 | } 179 | 180 | AbstractUpstreamHandShaker handShaker = ProtocolManager.createHandShaker( 181 | outboundProtocol, session, channel, activeProxyIp, 182 | value -> { 183 | if (value.isSuccess()) { 184 | recorder.recordEvent(() -> "HandShark success"); 185 | callback.onHandSharkFinished(channel, outboundProtocol); 186 | 187 | session.getProxyServer().getProxyCompose().markSessionUse(session, activeProxyIp); 188 | return; 189 | } 190 | channel.close(); 191 | if (value.v) { 192 | // 可以被重试，那么走重试逻辑 193 | proxyForward(value.e); 194 | return; 195 | } 196 | callback.onHandSharkError(value.e); 197 | }); 198 | 199 | recorder.recordMosaicMsgIfSubscribeRecorder(() -> "begin HandShark with channel: " + channel); 200 | handShaker.doHandShark(); 201 | }; 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/proxy/switcher/UpstreamHandSharkCallback.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.proxy.switcher; 2 | 3 | import cn.iinti.proxycompose.proxy.outbound.handshark.Protocol; 4 | import io.netty.channel.Channel; 5 | 6 | import javax.annotation.Nullable; 7 | 8 | public interface UpstreamHandSharkCallback { 9 | void onHandSharkFinished(Channel upstreamChannel, @Nullable Protocol outboundProtocol); 10 | 11 | void onHandSharkError(Throwable e); 12 | } -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/resource/DropReason.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.resource; 2 | 3 | public enum DropReason { 4 | /** 5 | * IP入库前连通性检查失败 6 | */ 7 | IP_TEST_FAILED, 8 | /** 9 | * 使用期间，发现ip资源服务不可用。 10 | */ 11 | IP_SERVER_UNAVAILABLE, 12 | /** 13 | * 使用期间动态判断ip资源质量差 14 | */ 15 | IP_QUALITY_BAD, 16 | /** 17 | * IP没有问题，但空转无使用，直到更加新的IP资源入库替换本资源 18 | */ 19 | IP_IDLE_POOL_OVERFLOW, 20 | /** 21 | * IP没有问题，但使用时间达到用户配置/指定的存活时间而被自动下线 22 | */ 23 | IP_ALIVE_TIME_REACHED, 24 | /** 25 | * 管理员关闭整个IP池资源，此时下线本ip池中所有的IP资源 26 | */ 27 | IP_RESOURCE_CLOSE, 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/resource/IpAndPort.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.resource; 2 | 3 | import lombok.Getter; 4 | 5 | @Getter 6 | public class IpAndPort { 7 | private final String ip; 8 | private final Integer port; 9 | private final String ipPort; 10 | 11 | 12 | public IpAndPort(String ip, Integer port) { 13 | this.ip = ip; 14 | this.port = port; 15 | this.ipPort = ip + ":" + port; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return ipPort; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/resource/IpResourceParser.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.resource; 2 | 3 | import com.alibaba.fastjson.JSONArray; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.google.common.collect.Lists; 6 | import org.apache.commons.lang3.StringUtils; 7 | 8 | import java.util.Collections; 9 | import java.util.List; 10 | 11 | public interface IpResourceParser { 12 | List parse(String responseText); 13 | 14 | public static IpResourceParser resolve(String format) { 15 | return "json".equalsIgnoreCase(format) ? 16 | JSONParser.instance : SmartParser.instance; 17 | } 18 | 19 | class JSONParser implements IpResourceParser { 20 | 21 | public static JSONParser instance = new JSONParser(); 22 | 23 | @Override 24 | public List parse(String responseText) { 25 | responseText = StringUtils.trimToEmpty(responseText); 26 | if (responseText.startsWith("[")) { 27 | return JSONArray.parseArray(responseText, ProxyIp.class); 28 | } 29 | if (responseText.startsWith("{")) { 30 | return Lists.newArrayList(JSONObject.parseObject(responseText, ProxyIp.class)); 31 | } 32 | return Collections.emptyList(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/resource/ProxyIp.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.resource; 2 | 3 | import cn.iinti.proxycompose.utils.IpUtils; 4 | import lombok.Data; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import javax.annotation.Nonnull; 8 | import javax.annotation.Nullable; 9 | import java.util.Comparator; 10 | 11 | /** 12 | * 一个真实的代理ip资源 13 | * 14 | * @see IpUtils#check(ProxyIp) 15 | */ 16 | @Data 17 | public class ProxyIp implements Comparable { 18 | 19 | private String proxyHost; 20 | 21 | 22 | private Integer proxyPort; 23 | 24 | 25 | // 以下为扩展内容,他不是必须的，用以应对各种特殊的ip供应商 26 | // (如芝麻代理，他ip有有效期，但是过期不拒绝，反而在http层返回错误内容，导致框架无法感知) 27 | 28 | /** 29 | * 鉴权用户名称，一般应该使用整个ipsource的鉴权 30 | */ 31 | private String userName; 32 | /** 33 | * 鉴权用户密码，一般应该使用整个ipSource的鉴权 34 | */ 35 | private String password; 36 | 37 | /** 38 | * 该ip资源下线时间戳，时区使用东八区 39 | */ 40 | private Long expireTime; 41 | 42 | /** 43 | * 出口ip：如果您关闭了ip连通性检查，则证明您的ip供应具备非常高的质量， 44 | * 此时您应该主动提供出口ip资源（即在ResourceHandler中手动设置本字段） 45 | * 请注意基于国家/城市的分发、基于经纬度的距离分发两种特性均强依赖出口ip解析。 46 | * 如果最终缺失本字段，则会导致上诉两种算法策略不生效 47 | */ 48 | @Nullable 49 | private String outIp; 50 | 51 | /** 52 | * 资源唯一ID，可选项，如扩展为空，则默认填充为-> ip:port 53 | */ 54 | private String resourceId; 55 | 56 | 57 | @Override 58 | public int compareTo(@Nonnull ProxyIp o) { 59 | int i = Comparator.comparing(ProxyIp::getProxyHost).compare(this, o); 60 | if (i == 0) { 61 | i = proxyPort.compareTo(o.proxyPort); 62 | } 63 | return i; 64 | } 65 | 66 | private String getIpPort() { 67 | return proxyHost + ":" + proxyPort; 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return getIpPort(); 73 | } 74 | 75 | /** 76 | * this method will be call by framework 77 | */ 78 | public ProxyIp resolveId() { 79 | if (StringUtils.isNotBlank(resourceId)) { 80 | return this; 81 | } 82 | resourceId = getIpPort(); 83 | return this; 84 | } 85 | 86 | public boolean isValid() { 87 | return StringUtils.isNotBlank(proxyHost) && proxyPort != null 88 | && proxyPort > 0 && proxyPort < 65535; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/resource/SmartParser.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.resource; 2 | 3 | import cn.iinti.proxycompose.utils.IpUtils; 4 | import com.google.common.base.CharMatcher; 5 | import com.google.common.base.Splitter; 6 | import com.google.common.collect.Lists; 7 | 8 | import java.util.List; 9 | import java.util.TreeSet; 10 | 11 | /** 12 | * 通用ip格式解析器，同时支持IpPortPlain和PortSpace 13 | */ 14 | public class SmartParser implements IpResourceParser { 15 | 16 | public static SmartParser instance = new SmartParser(); 17 | 18 | public static void main(String[] args) { 19 | instance.parse("182.244.169.248:57114\n" + 20 | "113.128.31.3:57114\n" + 21 | "36.102.173.123:57114\n" + 22 | "182.38.126.190:57114") 23 | .forEach(System.out::println); 24 | System.out.println("-------"); 25 | instance.parse("haproxy1.dailiyun.com:20000-20200,haproxy2.dailiyun.com:20200-20500") 26 | .forEach(System.out::println); 27 | } 28 | 29 | private static final Splitter smartSplitter = Splitter.on(CharMatcher.anyOf("\n,")).omitEmptyStrings().trimResults(); 30 | private static final Splitter portSplitter = Splitter.on(':').omitEmptyStrings().trimResults(); 31 | 32 | @Override 33 | public List parse(String responseText) { 34 | TreeSet treeSet = new TreeSet<>(); 35 | smartSplitter.split(responseText) 36 | .forEach(pair -> { 37 | List ipAndPortSpace = portSplitter.splitToList(pair); 38 | if (ipAndPortSpace.size() != 2) { 39 | return; 40 | } 41 | String ip = ipAndPortSpace.get(0); 42 | String portSpace = ipAndPortSpace.get(1); 43 | if (portSpace.contains("-")) { 44 | fillSpace(ip, portSpace, treeSet); 45 | } else { 46 | treeSet.add(IpUtils.fromIpPort(ip, Integer.parseInt(portSpace))); 47 | } 48 | }); 49 | return Lists.newArrayList(treeSet); 50 | } 51 | 52 | private static void fillSpace(String ip, String portSpace, TreeSet treeSet) { 53 | int index = portSpace.indexOf("-"); 54 | String startStr = portSpace.substring(0, index); 55 | String endStr = portSpace.substring(index + 1); 56 | int start = Integer.parseInt(startStr); 57 | int end = Integer.parseInt(endStr); 58 | for (int i = start; i <= end; i++) { 59 | treeSet.add(IpUtils.fromIpPort(ip, i)); 60 | } 61 | } 62 | 63 | private SmartParser() { 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/trace/Recorder.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.trace; 2 | 3 | 4 | 5 | import cn.iinti.proxycompose.loop.Looper; 6 | import cn.iinti.proxycompose.trace.impl.SubscribeRecorders; 7 | import cn.iinti.proxycompose.trace.utils.StringSplitter; 8 | import cn.iinti.proxycompose.trace.utils.ThrowablePrinter; 9 | 10 | import java.util.Collection; 11 | import java.util.LinkedList; 12 | import java.util.function.Function; 13 | 14 | public abstract class Recorder { 15 | protected static final Looper thread = new Looper("eventRecorder").startLoop(); 16 | 17 | public static Looper workThread() { 18 | return thread; 19 | } 20 | 21 | 22 | public void recordEvent(String message) { 23 | recordEvent(() -> message, (Throwable) null); 24 | } 25 | 26 | public void recordEvent(String message, Throwable throwable) { 27 | recordEvent(() -> message, throwable); 28 | } 29 | 30 | public void recordEvent(MessageGetter messageGetter) { 31 | recordEvent(messageGetter, (Throwable) null); 32 | } 33 | 34 | public void recordEvent(T t, Function fuc) { 35 | recordEvent(() -> fuc.apply(t)); 36 | } 37 | 38 | public abstract void recordEvent(MessageGetter messageGetter, Throwable throwable); 39 | 40 | public void recordMosaicMsgIfSubscribeRecorder(MessageGetter message) { 41 | if (this instanceof SubscribeRecorders.SubscribeRecorder) { 42 | SubscribeRecorders.SubscribeRecorder s = (SubscribeRecorders.SubscribeRecorder) this; 43 | s.recordMosaicMsg(message); 44 | } else { 45 | recordEvent(message); 46 | } 47 | } 48 | 49 | protected Collection splitMsg(String msg, Throwable throwable) { 50 | Collection strings = StringSplitter.split(msg, '\n'); 51 | if (throwable == null) { 52 | return strings; 53 | } 54 | if (strings.isEmpty()) { 55 | // 确保可以被编辑 56 | strings = new LinkedList<>(); 57 | } 58 | ThrowablePrinter.printStackTrace(strings, throwable); 59 | return strings; 60 | } 61 | 62 | public interface MessageGetter { 63 | String getMessage(); 64 | } 65 | 66 | public static final Recorder nop = new Recorder() { 67 | @Override 68 | public void recordEvent(MessageGetter messageGetter, Throwable throwable) { 69 | 70 | } 71 | }; 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/trace/impl/DiskRecorders.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.trace.impl; 2 | 3 | import cn.iinti.proxycompose.trace.Recorder; 4 | import lombok.Getter; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.slf4j.MDC; 8 | 9 | import java.util.Collection; 10 | import java.util.Collections; 11 | import java.util.LinkedList; 12 | import java.util.List; 13 | import java.util.function.Consumer; 14 | 15 | public class DiskRecorders { 16 | 17 | public static DiskRecorders USER_SESSION = new DiskRecorders("user", false); 18 | /** 19 | * 给debug调试使用，传递本参数可以打印全流程日志 20 | */ 21 | public static DiskRecorders USER_DEBUG_TRACE = new DiskRecorders("user_trace", true); 22 | public static DiskRecorders IP_SOURCE = new DiskRecorders("ip_source", true); 23 | public static DiskRecorders IP_TEST = new DiskRecorders("ip_test", false); 24 | public static DiskRecorders MOCK_SESSION = new DiskRecorders("mock_proxy", false); 25 | public static DiskRecorders PORTAL = new DiskRecorders("portal", false); 26 | public static DiskRecorders OTHER = new DiskRecorders("other", false); 27 | 28 | private static final Logger log = LoggerFactory.getLogger("EventTrace"); 29 | 30 | @Getter 31 | private final String tag; 32 | @Getter 33 | private final boolean all; 34 | private final WheelSlotFilter wheelSlotFilter; 35 | 36 | 37 | public DiskRecorders(String tag, boolean all) { 38 | this.tag = tag; 39 | this.all = all; 40 | this.wheelSlotFilter = new WheelSlotFilter(all); 41 | } 42 | 43 | public DiskRecorder acquireRecorder(String sessionId, boolean debug) { 44 | return this.wheelSlotFilter.acquireRecorder(debug) ? 45 | new DiskRecorderImpl(sessionId) : nopDiskRecorder; 46 | } 47 | 48 | public static abstract class DiskRecorder extends Recorder { 49 | abstract void takeHistory(Consumer> consumer); 50 | 51 | abstract void recordBatchEvent(Collection msgLines); 52 | 53 | abstract boolean enable(); 54 | } 55 | 56 | private class DiskRecorderImpl extends DiskRecorder { 57 | private final String sessionId; 58 | 59 | private LinkedList historyMsg = new LinkedList<>(); 60 | private static final int MAX_HISTORY = 100; 61 | 62 | private DiskRecorderImpl(String sessionId) { 63 | this.sessionId = sessionId; 64 | } 65 | 66 | @Override 67 | void recordBatchEvent(Collection msgLines) { 68 | thread.execute(() -> { 69 | MDC.put("Scene", tag); 70 | historyMsg.addAll(msgLines); 71 | int overflow = historyMsg.size() - MAX_HISTORY; 72 | for (int i = 0; i < overflow; i++) { 73 | historyMsg.removeFirst(); 74 | } 75 | msgLines.forEach(s -> log.info("sessionId:" + sessionId + " -> " + s)); 76 | }); 77 | } 78 | 79 | @Override 80 | public void recordEvent(MessageGetter messageGetter, Throwable throwable) { 81 | recordBatchEvent(splitMsg(messageGetter.getMessage(), throwable)); 82 | } 83 | 84 | public void takeHistory(Consumer> consumer) { 85 | thread.execute(() -> { 86 | LinkedList historySnapshot = historyMsg; 87 | historyMsg = new LinkedList<>(); 88 | consumer.accept(historySnapshot); 89 | }); 90 | } 91 | 92 | boolean enable() { 93 | return true; 94 | } 95 | } 96 | 97 | 98 | private final static DiskRecorder nopDiskRecorder = new DiskRecorder() { 99 | public void takeHistory(Consumer> consumer) { 100 | consumer.accept(Collections.emptyList()); 101 | } 102 | 103 | @Override 104 | public void recordEvent(MessageGetter messageGetter, Throwable throwable) { 105 | 106 | } 107 | 108 | void recordBatchEvent(Collection msgLines) { 109 | 110 | } 111 | 112 | boolean enable() { 113 | return false; 114 | } 115 | }; 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/trace/impl/SubscribeRecorders.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.trace.impl; 2 | 3 | import cn.iinti.proxycompose.trace.Recorder; 4 | import com.google.common.collect.HashMultimap; 5 | import com.google.common.collect.Lists; 6 | import com.google.common.collect.Maps; 7 | import com.google.common.collect.Multimap; 8 | import lombok.Getter; 9 | 10 | import java.util.Collection; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.UUID; 14 | import java.util.function.Consumer; 15 | 16 | public class SubscribeRecorders { 17 | private static final Map recorderCategories = Maps.newHashMap(); 18 | 19 | public static SubscribeRecorders fromTag( String tag) { 20 | return recorderCategories.get(tag.toLowerCase()); 21 | } 22 | 23 | public static SubscribeRecorders USER_SESSION = new SubscribeRecorders(DiskRecorders.USER_SESSION, false); 24 | public static SubscribeRecorders USER_DEBUG_TRACE = new SubscribeRecorders(DiskRecorders.USER_DEBUG_TRACE, true); 25 | public static SubscribeRecorders IP_SOURCE = new SubscribeRecorders(DiskRecorders.IP_SOURCE, true); 26 | public static SubscribeRecorders IP_TEST = new SubscribeRecorders(DiskRecorders.IP_TEST, true); 27 | public static SubscribeRecorders MOCK_SESSION = new SubscribeRecorders(DiskRecorders.MOCK_SESSION, true); 28 | public static SubscribeRecorders PORTAL = new SubscribeRecorders(DiskRecorders.PORTAL, true); 29 | public static SubscribeRecorders OTHER = new SubscribeRecorders(DiskRecorders.OTHER, false); 30 | 31 | 32 | private final Multimap listenerRegistry = HashMultimap.create(); 33 | private final Map scopeSlotFilters = Maps.newHashMap(); 34 | private final DiskRecorders lowLevel; 35 | 36 | @Getter 37 | private final String tag; 38 | 39 | 40 | private final boolean lowLevelAll; 41 | 42 | @Getter 43 | private final boolean forAdmin; 44 | 45 | public SubscribeRecorders(DiskRecorders lowLevel, boolean forAdmin) { 46 | this.lowLevel = lowLevel; 47 | this.forAdmin = forAdmin; 48 | this.tag = lowLevel.getTag().toLowerCase(); 49 | this.lowLevelAll = lowLevel.isAll(); 50 | recorderCategories.put(tag, this); 51 | } 52 | 53 | public void registerListener(String scope, Listener listener) { 54 | Recorder.workThread().execute(() -> listenerRegistry.put(scope, listener)); 55 | } 56 | 57 | public void unregisterListener(String scope, Listener listener) { 58 | Recorder.workThread().execute(() -> listenerRegistry.remove(scope, listener)); 59 | } 60 | 61 | public SubscribeRecorder acquireRecorder(boolean debug) { 62 | return acquireRecorder(debug, "default"); 63 | } 64 | 65 | public SubscribeRecorder acquireRecorder(boolean debug, String scope) { 66 | return acquireRecorder(UUID.randomUUID().toString(), debug, scope); 67 | } 68 | 69 | public SubscribeRecorder acquireRecorder(String sessionId, boolean debug, String scope) { 70 | return new SubscribeRecorder(sessionId, lowLevel.acquireRecorder(sessionId, debug), scope); 71 | } 72 | 73 | public class SubscribeRecorder extends Recorder { 74 | 75 | private final DiskRecorders.DiskRecorder lowLevel; 76 | private String[] scopes; 77 | private boolean subscribeTicket; 78 | private final String sessionId; 79 | 80 | 81 | public SubscribeRecorder(String sessionId, DiskRecorders.DiskRecorder lowLevel, String scope) { 82 | this.lowLevel = lowLevel; 83 | this.sessionId = sessionId; 84 | this.scopes = new String[]{scope}; 85 | refreshSlotTick(); 86 | } 87 | 88 | private void refreshSlotTick() { 89 | if (lowLevelAll) { 90 | subscribeTicket = true; 91 | return; 92 | } 93 | Recorder.workThread().execute(() -> { 94 | boolean hint = false; 95 | for (String scope : scopes) { 96 | boolean ticket = scopeSlotFilters 97 | .computeIfAbsent(scope, (k) -> new WheelSlotFilter(false)) 98 | .acquireRecorder(false); 99 | if (ticket) { 100 | hint = true; 101 | } 102 | } 103 | subscribeTicket = hint; 104 | }); 105 | 106 | } 107 | 108 | public void takeHistory(Consumer> consumer) { 109 | lowLevel.takeHistory(consumer); 110 | } 111 | 112 | public void recordBatchEvent(Collection msgLines) { 113 | lowLevel.recordBatchEvent(msgLines); 114 | } 115 | 116 | @Override 117 | public void recordEvent(MessageGetter messageGetter, Throwable throwable) { 118 | List targetListenerList = Lists.newArrayList(); 119 | for (String scope : scopes) { 120 | Collection listeners = listenerRegistry.get(scope); 121 | targetListenerList.addAll(listeners); 122 | } 123 | 124 | if (!lowLevel.enable() && (targetListenerList.isEmpty() || !subscribeTicket)) { 125 | // this indicate no log print need 126 | return; 127 | } 128 | 129 | thread.execute(() -> { 130 | Collection msgLines = splitMsg(messageGetter.getMessage(), throwable); 131 | lowLevel.recordBatchEvent(msgLines); 132 | 133 | if (subscribeTicket) { 134 | targetListenerList.forEach(listener -> msgLines.forEach(s -> listener.onLogMsg(sessionId, s))); 135 | } 136 | }); 137 | } 138 | 139 | public void changeScope(String... scope) { 140 | this.scopes = scope; 141 | refreshSlotTick(); 142 | } 143 | 144 | 145 | public void recordMosaicMsg(MessageGetter message) { 146 | recordEvent(() -> NOT_SHOW_FOR_NORMAL_USER + message.getMessage()); 147 | } 148 | 149 | 150 | } 151 | 152 | private static final String NOT_SHOW_FOR_NORMAL_USER = "nsfnu:"; 153 | 154 | public static boolean isMosaicMsg(String msg) { 155 | return msg.startsWith(NOT_SHOW_FOR_NORMAL_USER); 156 | } 157 | 158 | public interface Listener { 159 | void onLogMsg(String sessionId, String line); 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/trace/impl/WheelSlotFilter.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.trace.impl; 2 | 3 | import java.util.ArrayList; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | public class WheelSlotFilter { 7 | 8 | private final ArrayList slots; 9 | private final boolean all; 10 | 11 | public WheelSlotFilter(boolean all) { 12 | this.all = all; 13 | this.slots = new ArrayList<>(30); 14 | for (int i = 0; i < 30; i++) { 15 | this.slots.add(new AtomicLong()); 16 | } 17 | } 18 | 19 | public boolean acquireRecorder(boolean debug) { 20 | if (all || debug) { 21 | return true; 22 | } 23 | long nowTime = System.currentTimeMillis(); 24 | int slotIndex = ((int) ((nowTime / 1000) % 60)) / 2; 25 | long timeMinute = nowTime / 60000; 26 | 27 | AtomicLong slot = slots.get(slotIndex); 28 | 29 | long slotTime = slot.get(); 30 | if (slotTime == timeMinute) { 31 | return false; 32 | } 33 | return slot.compareAndSet(slotTime, timeMinute); 34 | } 35 | 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/trace/utils/ThrowablePrinter.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.trace.utils; 2 | 3 | import java.util.Collection; 4 | import java.util.Collections; 5 | import java.util.IdentityHashMap; 6 | import java.util.Set; 7 | 8 | public class ThrowablePrinter { 9 | /** 10 | * Caption for labeling causative exception stack traces 11 | */ 12 | private static final String CAUSE_CAPTION = "Caused by: "; 13 | 14 | /** 15 | * Caption for labeling suppressed exception stack traces 16 | */ 17 | private static final String SUPPRESSED_CAPTION = "Suppressed: "; 18 | 19 | 20 | 21 | public static void printStackTrace(Collection out, Throwable throwable) { 22 | // Guard against malicious overrides of Throwable.equals by 23 | // using a Set with identity equality semantics. 24 | Set dejaVu = 25 | Collections.newSetFromMap(new IdentityHashMap()); 26 | dejaVu.add(throwable); 27 | 28 | 29 | // Print our stack trace 30 | //s.println(this); 31 | out.add(throwable.toString()); 32 | StackTraceElement[] trace = throwable.getStackTrace(); 33 | for (StackTraceElement traceElement : trace) 34 | out.add("\tat " + traceElement); 35 | 36 | // Print suppressed exceptions, if any 37 | for (Throwable se : throwable.getSuppressed()) 38 | printEnclosedStackTrace(out, se, trace, SUPPRESSED_CAPTION, "\t", dejaVu); 39 | 40 | // Print cause, if any 41 | Throwable ourCause = throwable.getCause(); 42 | if (ourCause != null) 43 | printEnclosedStackTrace(out, ourCause, trace, CAUSE_CAPTION, "", dejaVu); 44 | 45 | } 46 | 47 | /** 48 | * Print our stack trace as an enclosed exception for the specified 49 | * stack trace. 50 | */ 51 | private static void printEnclosedStackTrace(Collection out, Throwable throwable, 52 | StackTraceElement[] enclosingTrace, 53 | String caption, 54 | String prefix, 55 | Set dejaVu) { 56 | 57 | if (dejaVu.contains(throwable)) { 58 | out.add("\t[CIRCULAR REFERENCE:" + throwable + "]"); 59 | } else { 60 | dejaVu.add(throwable); 61 | // Compute number of frames in common between this and enclosing trace 62 | StackTraceElement[] trace = throwable.getStackTrace(); 63 | int m = trace.length - 1; 64 | int n = enclosingTrace.length - 1; 65 | while (m >= 0 && n >= 0 && trace[m].equals(enclosingTrace[n])) { 66 | m--; 67 | n--; 68 | } 69 | int framesInCommon = trace.length - 1 - m; 70 | 71 | // Print our stack trace 72 | out.add(prefix + caption + throwable); 73 | for (int i = 0; i <= m; i++) 74 | out.add(prefix + "\tat " + trace[i]); 75 | if (framesInCommon != 0) 76 | out.add(prefix + "\t... " + framesInCommon + " more"); 77 | 78 | // Print suppressed exceptions, if any 79 | for (Throwable se : throwable.getSuppressed()) 80 | printEnclosedStackTrace(out, se, trace, SUPPRESSED_CAPTION, 81 | prefix + "\t", dejaVu); 82 | 83 | // Print cause, if any 84 | Throwable ourCause = throwable.getCause(); 85 | if (ourCause != null) 86 | printEnclosedStackTrace(out, ourCause, trace, CAUSE_CAPTION, prefix, dejaVu); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/AsyncHttpInvoker.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import cn.iinti.proxycompose.loop.ValueCallback; 4 | import cn.iinti.proxycompose.trace.Recorder; 5 | import org.asynchttpclient.AsyncHttpClient; 6 | import org.asynchttpclient.BoundRequestBuilder; 7 | import org.asynchttpclient.DefaultAsyncHttpClientConfig; 8 | import org.asynchttpclient.proxy.ProxyServer; 9 | 10 | import java.nio.charset.StandardCharsets; 11 | 12 | import static org.asynchttpclient.Dsl.asyncHttpClient; 13 | 14 | /** 15 | * 请注意，为了避免日志刷屏，本模块强制要求使用recorder记录日志 16 | * 另外由于他是异步的，也要求使用recorder 17 | */ 18 | public class AsyncHttpInvoker { 19 | public static final AsyncHttpClient httpclient = asyncHttpClient( 20 | new DefaultAsyncHttpClientConfig.Builder() 21 | .setKeepAlive(true) 22 | .setConnectTimeout(10000) 23 | .setReadTimeout(8000) 24 | .setPooledConnectionIdleTimeout(20000) 25 | .setEventLoopGroup(NettyThreadPools.asyncHttpWorkGroup) 26 | .build()); 27 | 28 | 29 | 30 | public static void get(String url, Recorder recorder, ValueCallback callback) { 31 | get(url, recorder, null, callback); 32 | } 33 | 34 | public static void get(String url, Recorder recorder, ProxyServer.Builder proxyBuilder, ValueCallback callback) { 35 | recorder.recordEvent(() -> "begin async invoker: " + url); 36 | BoundRequestBuilder getRequest = httpclient.prepareGet(url); 37 | if (proxyBuilder != null) { 38 | getRequest.setProxyServer(proxyBuilder); 39 | } 40 | 41 | execute(getRequest, recorder, callback); 42 | } 43 | 44 | private static void execute(BoundRequestBuilder requestBuilder, Recorder recorder, ValueCallback callback) { 45 | requestBuilder.execute().toCompletableFuture() 46 | .whenCompleteAsync((response, throwable) -> { 47 | if (throwable != null) { 48 | ValueCallback.failed(callback, throwable); 49 | return; 50 | } 51 | try { 52 | String responseBody = response.getResponseBody(StandardCharsets.UTF_8).trim(); 53 | recorder.recordEvent(() -> "async response:" + responseBody); 54 | ValueCallback.success(callback, responseBody); 55 | } catch (Exception e) { 56 | recorder.recordEvent(() -> "read response failed", e); 57 | ValueCallback.failed(callback, e); 58 | } 59 | }); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/ConsistentHashUtil.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | 4 | import com.google.common.hash.Hashing; 5 | 6 | import java.nio.charset.StandardCharsets; 7 | import java.util.Iterator; 8 | import java.util.Map; 9 | import java.util.SortedMap; 10 | import java.util.TreeMap; 11 | 12 | public class ConsistentHashUtil { 13 | public static long murHash(String key) { 14 | return Hashing.goodFastHash(128) 15 | .hashString(key, StandardCharsets.UTF_8).asLong(); 16 | } 17 | 18 | public static T fetchConsistentRing(TreeMap treeMap, K key) { 19 | if (treeMap.isEmpty()) { 20 | return null; 21 | } 22 | SortedMap tailMap = treeMap.tailMap(key); 23 | if (tailMap.isEmpty()) { 24 | return treeMap.values().iterator().next(); 25 | } 26 | return tailMap.values().iterator().next(); 27 | } 28 | 29 | public static Iterable constantRingIt(TreeMap treeMap, K key) { 30 | Iterator> tailIterator = treeMap.tailMap(key).entrySet().iterator(); 31 | Iterator> headIterator = treeMap.entrySet().iterator(); 32 | return () -> new Iterator() { 33 | private T value; 34 | 35 | @Override 36 | public boolean hasNext() { 37 | if (tailIterator.hasNext()) { 38 | value = tailIterator.next().getValue(); 39 | return true; 40 | } 41 | if (headIterator.hasNext()) { 42 | Map.Entry next = headIterator.next(); 43 | K nextKey = next.getKey(); 44 | if (treeMap.comparator().compare(nextKey, key) >= 0) { 45 | value = null; 46 | return false; 47 | } 48 | value = next.getValue(); 49 | return true; 50 | 51 | } 52 | return false; 53 | } 54 | 55 | @Override 56 | public T next() { 57 | return value; 58 | } 59 | }; 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/IniConfig.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import com.alibaba.fastjson.parser.ParserConfig; 4 | import com.alibaba.fastjson.util.TypeUtils; 5 | import lombok.SneakyThrows; 6 | import org.apache.commons.lang3.StringUtils; 7 | import org.ini4j.ConfigParser; 8 | 9 | import java.lang.reflect.ParameterizedType; 10 | import java.lang.reflect.Type; 11 | import java.util.function.Consumer; 12 | 13 | public abstract class IniConfig { 14 | private final ConfigParser config; 15 | protected final String section; 16 | 17 | public IniConfig(ConfigParser config, String section) { 18 | this.config = config; 19 | this.section = section; 20 | } 21 | 22 | @SneakyThrows 23 | protected void acceptConfig(String key, Consumer consumer) { 24 | if (config.hasOption(section, key)) { 25 | consumer.accept(config.get(section, key)); 26 | } 27 | } 28 | 29 | public abstract class ConfigValue { 30 | public final V value; 31 | public final String key; 32 | 33 | public ConfigValue(String key, V defaultValue) { 34 | this.key = key; 35 | this.value = calcValue(key, defaultValue); 36 | } 37 | 38 | @SneakyThrows 39 | private V calcValue(String key, V defaultValue) { 40 | key = key.toLowerCase(); 41 | Class superClassGenericType = getSuperClassGenericType(getClass()); 42 | if (config.hasOption(section, key)) { 43 | String config = IniConfig.this.config.get(section, key); 44 | if (StringUtils.isNotBlank(config)) { 45 | return TypeUtils.cast(config, superClassGenericType, ParserConfig.getGlobalInstance()); 46 | } 47 | } 48 | return defaultValue; 49 | } 50 | 51 | @SuppressWarnings("unchecked") 52 | private Class getSuperClassGenericType(Class clazz) { 53 | Type genType = clazz.getGenericSuperclass(); 54 | if (!(genType instanceof ParameterizedType)) { 55 | return (Class) Object.class; 56 | } 57 | Type[] params = ((ParameterizedType) genType).getActualTypeArguments(); 58 | if (0 == params.length) { 59 | return (Class) Object.class; 60 | } else if (!(params[0] instanceof Class)) { 61 | return (Class) Object.class; 62 | } else { 63 | return (Class) params[0]; 64 | } 65 | } 66 | } 67 | 68 | 69 | public class StringConfigValue extends ConfigValue { 70 | public StringConfigValue(String configKey, String defaultValue) { 71 | super(configKey, defaultValue); 72 | } 73 | } 74 | 75 | public class BooleanConfigValue extends ConfigValue { 76 | public BooleanConfigValue(String configKey, Boolean defaultValue) { 77 | super(configKey, defaultValue); 78 | } 79 | } 80 | 81 | public class IntegerConfigValue extends ConfigValue { 82 | public IntegerConfigValue(String configKey, Integer defaultValue) { 83 | super(configKey, defaultValue); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/IpUtils.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import cn.iinti.proxycompose.resource.ProxyIp; 4 | import com.google.common.base.Splitter; 5 | import org.apache.commons.lang3.StringUtils; 6 | import org.apache.commons.lang3.math.NumberUtils; 7 | 8 | import java.net.*; 9 | import java.util.Enumeration; 10 | import java.util.List; 11 | 12 | public class IpUtils { 13 | 14 | public static ProxyIp fromIpPort(String ip, int port) { 15 | ProxyIp proxyIp = new ProxyIp(); 16 | proxyIp.setProxyHost(ip); 17 | proxyIp.setProxyPort(port); 18 | return proxyIp; 19 | } 20 | 21 | 22 | public static String check(ProxyIp proxyIp) { 23 | String ip = proxyIp.getProxyHost(); 24 | if (StringUtils.isBlank(ip)) { 25 | return "ip can not empty"; 26 | } 27 | ip = ip.trim(); 28 | proxyIp.setProxyHost(ip); 29 | if (!isIpV4(ip)) { 30 | try { 31 | InetAddress byName = InetAddress.getByName(ip); 32 | } catch (UnknownHostException e) { 33 | return "error domain:" + e.getMessage(); 34 | } 35 | } 36 | 37 | Integer port = proxyIp.getProxyPort(); 38 | if (port == null || port <= 0 || port > 65535) { 39 | return "port range error"; 40 | } 41 | 42 | return null; 43 | } 44 | 45 | private static final Splitter dotSplitter = Splitter.on('.'); 46 | 47 | public static boolean isIpV4(String input) { 48 | if (StringUtils.isBlank(input)) { 49 | return false; 50 | } 51 | // 3 * 4 + 3 = 15 52 | // 1 * 4 + 3 = 7 53 | if (input.length() > 15 || input.length() < 7) { 54 | return false; 55 | } 56 | 57 | List split = dotSplitter.splitToList(input); 58 | if (split.size() != 4) { 59 | return false; 60 | } 61 | for (String segment : split) { 62 | int i = NumberUtils.toInt(segment, -1); 63 | if (i < 0 || i > 255) { 64 | return false; 65 | } 66 | } 67 | return true; 68 | } 69 | 70 | public static boolean isValidIp(String ip) { 71 | return StringUtils.isNotBlank(ip) && isIpV4(ip); 72 | } 73 | 74 | 75 | public static String fetchIp(String type) throws SocketException { 76 | Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); 77 | while (networkInterfaces.hasMoreElements()) { 78 | NetworkInterface networkInterface = networkInterfaces.nextElement(); 79 | if (networkInterface.isLoopback()) { 80 | continue; 81 | } 82 | 83 | Enumeration inetAddresses = networkInterface.getInetAddresses(); 84 | while (inetAddresses.hasMoreElements()) { 85 | InetAddress inetAddress = inetAddresses.nextElement(); 86 | if (inetAddress instanceof Inet6Address) { 87 | continue; 88 | } 89 | Inet4Address inet4Address = (Inet4Address) inetAddress; 90 | byte[] address = inet4Address.getAddress(); 91 | if (address.length != 4) { 92 | continue; 93 | } 94 | int firstByte = address[0] & 0xFF; 95 | boolean isPrivate = (firstByte == 192 || firstByte == 10 || firstByte == 172); 96 | if (type.equals("private")) { 97 | if (isPrivate) { 98 | return inet4Address.getHostAddress(); 99 | } 100 | } else { 101 | if (!isPrivate) { 102 | return inet4Address.getHostAddress(); 103 | } 104 | } 105 | } 106 | } 107 | return null; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/NettyThreadPools.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import io.netty.channel.nio.NioEventLoopGroup; 4 | import io.netty.util.concurrent.DefaultThreadFactory; 5 | 6 | public class NettyThreadPools { 7 | 8 | public static final NioEventLoopGroup outboundGroup = newDefaultEventLoop("outbound"); 9 | 10 | public static final NioEventLoopGroup proxyServerBossGroup = newDefaultEventLoop("Proxy-boss-group"); 11 | public static final NioEventLoopGroup proxyServerWorkerGroup = newDefaultEventLoop("Proxy-worker-group"); 12 | 13 | // 下载代理，通知外部等业务http调用 14 | public static final NioEventLoopGroup asyncHttpWorkGroup = 15 | new NioEventLoopGroup(1, createThreadFactory("async-http-invoker")); 16 | //newDefaultEventLoop("async-http-invoker"); 17 | 18 | 19 | private static NioEventLoopGroup newDefaultEventLoop(String name) { 20 | return new NioEventLoopGroup(0, createThreadFactory(name)); 21 | } 22 | 23 | private static DefaultThreadFactory createThreadFactory(String name) { 24 | return new DefaultThreadFactory(name + "-" + DefaultThreadFactory.toPoolName(NioEventLoopGroup.class)); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/NettyUtil.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import cn.iinti.proxycompose.resource.IpAndPort; 4 | import io.netty.buffer.ByteBuf; 5 | import io.netty.buffer.Unpooled; 6 | import io.netty.channel.Channel; 7 | import io.netty.channel.ChannelFuture; 8 | import io.netty.channel.ChannelFutureListener; 9 | import io.netty.channel.ChannelHandlerContext; 10 | import io.netty.handler.codec.http.*; 11 | import org.apache.commons.lang3.math.NumberUtils; 12 | 13 | import java.nio.charset.StandardCharsets; 14 | import java.util.ArrayList; 15 | import java.util.Collection; 16 | import java.util.Collections; 17 | import java.util.List; 18 | 19 | import static io.netty.buffer.ByteBufUtil.appendPrettyHexDump; 20 | import static io.netty.util.internal.StringUtil.NEWLINE; 21 | 22 | public class NettyUtil { 23 | 24 | public static String throwableMsg(Throwable throwable) { 25 | String msg = throwable.getClass().getName() + ":" + throwable.getMessage(); 26 | StackTraceElement[] stackTrace = throwable.getStackTrace(); 27 | if (stackTrace.length > 0) { 28 | msg += "\n" + stackTrace[0].toString(); 29 | } 30 | return msg; 31 | } 32 | 33 | public static void loveOther(Channel girl, Channel boy) { 34 | girl.closeFuture().addListener((ChannelFutureListener) future -> boy.close()); 35 | boy.closeFuture().addListener((ChannelFutureListener) future -> girl.close()); 36 | } 37 | 38 | 39 | /** 40 | * Closes the specified channel after all queued write requests are flushed. 41 | */ 42 | public static void closeOnFlush(Channel channel) { 43 | if (channel.isActive()) { 44 | channel.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE); 45 | } 46 | } 47 | 48 | public static void closeIfActive(Channel channel) { 49 | if (channel == null) { 50 | return; 51 | } 52 | channel.close(); 53 | } 54 | 55 | public static void closeAll(Collection channels) { 56 | if (channels == null) { 57 | return; 58 | } 59 | for (Channel channel : channels) { 60 | closeIfActive(channel); 61 | } 62 | } 63 | 64 | public static void httpResponseText(Channel httpChannel, HttpResponseStatus status, String body) { 65 | DefaultFullHttpResponse response; 66 | if (body != null) { 67 | byte[] bytes = body.getBytes(StandardCharsets.UTF_8); 68 | ByteBuf content = Unpooled.copiedBuffer(bytes); 69 | response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status, content); 70 | response.headers().set(HttpHeaders.Names.CONTENT_LENGTH, bytes.length); 71 | response.headers().set(HttpHeaders.Names.CONTENT_TYPE, "text/html; charset=utf-8"); 72 | } else { 73 | response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, status); 74 | } 75 | HttpUtil.setKeepAlive(response, false); 76 | httpChannel.writeAndFlush(response) 77 | .addListener(ChannelFutureListener.CLOSE); 78 | } 79 | 80 | public static void addVia(HttpMessage httpMessage, String alias) { 81 | String newViaHeader = String.valueOf(httpMessage.protocolVersion().majorVersion()) + 82 | '.' + 83 | httpMessage.protocolVersion().minorVersion() + 84 | ' ' + 85 | alias; 86 | 87 | List vias; 88 | if (httpMessage.headers().contains(HttpHeaderNames.VIA)) { 89 | List existingViaHeaders = httpMessage.headers().getAll(HttpHeaderNames.VIA); 90 | vias = new ArrayList<>(existingViaHeaders); 91 | vias.add(newViaHeader); 92 | } else { 93 | vias = Collections.singletonList(newViaHeader); 94 | } 95 | 96 | httpMessage.headers().set(HttpHeaderNames.VIA, vias); 97 | } 98 | 99 | public static void sendHttpResponse(ChannelHandlerContext ctx, HttpRequest req, DefaultFullHttpResponse res) { 100 | res.headers().set(HttpHeaderNames.CONTENT_LENGTH, res.content().readableBytes()); 101 | ChannelFuture f = ctx.channel().writeAndFlush(res); 102 | if (!HttpUtil.isKeepAlive(req) || res.status().code() != 200) { 103 | f.addListener(ChannelFutureListener.CLOSE); 104 | } 105 | } 106 | 107 | 108 | 109 | /** 110 | * 打印ByteBuf的好工具 111 | */ 112 | public static String formatByteBuf(ChannelHandlerContext ctx, String eventName, ByteBuf msg) { 113 | String chStr = ctx == null ? "debug" : ctx.channel().toString(); 114 | if (msg == null) { 115 | msg = Unpooled.EMPTY_BUFFER; 116 | } 117 | int length = msg.readableBytes(); 118 | if (length == 0) { 119 | return chStr + ' ' + eventName + ": 0B"; 120 | } else { 121 | int outputLength = chStr.length() + 1 + eventName.length() + 2 + 10 + 1; 122 | 123 | int rows = length / 16 + (length % 15 == 0 ? 0 : 1) + 4; 124 | int hexDumpLength = 2 + rows * 80; 125 | outputLength += hexDumpLength; 126 | 127 | StringBuilder buf = new StringBuilder(outputLength); 128 | buf.append(chStr).append(' ').append(eventName).append(": ").append(length).append('B'); 129 | 130 | buf.append(NEWLINE); 131 | appendPrettyHexDump(buf, msg); 132 | return buf.toString(); 133 | } 134 | } 135 | 136 | 137 | public static IpAndPort parseProxyTarget(HttpRequest httpRequest,boolean isHttps) { 138 | String host = httpRequest.headers().get(HttpHeaderNames.HOST); 139 | if (host == null && isHttps) { 140 | host = httpRequest.uri(); 141 | } 142 | 143 | if (host == null) { 144 | return null; 145 | } 146 | 147 | if (!host.contains(":")) { 148 | host += isHttps ? ":443" : ":80"; 149 | } 150 | 151 | String[] split = host.split(":"); 152 | 153 | int port = NumberUtils.toInt(split[1].trim(), -1); 154 | boolean illegal = port > 0 && port <= 65535; 155 | 156 | if (illegal) { 157 | return new IpAndPort(split[0], port); 158 | } 159 | return null; 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/PortSpaceParser.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import com.google.common.base.Splitter; 4 | import com.google.common.collect.Maps; 5 | 6 | import java.util.Map; 7 | import java.util.TreeSet; 8 | 9 | public class PortSpaceParser { 10 | private static final Map> cache = Maps.newConcurrentMap(); 11 | 12 | public static TreeSet parsePortSpace(String config) { 13 | TreeSet treeSet = cache.get(config); 14 | if (treeSet != null) { 15 | return new TreeSet<>(treeSet); 16 | } 17 | treeSet = parsePortSpaceImpl(config); 18 | cache.put(config, treeSet); 19 | return new TreeSet<>(treeSet); 20 | } 21 | 22 | public static TreeSet parsePortSpaceImpl(String config) { 23 | TreeSet copyOnWriteTreeSet = new TreeSet<>(); 24 | Iterable pairs = Splitter.on(":").split(config); 25 | for (String pair : pairs) { 26 | if (pair.contains("-")) { 27 | int index = pair.indexOf("-"); 28 | String startStr = pair.substring(0, index); 29 | String endStr = pair.substring(index + 1); 30 | int start = Integer.parseInt(startStr); 31 | int end = Integer.parseInt(endStr); 32 | for (int i = start; i <= end; i++) { 33 | copyOnWriteTreeSet.add(i); 34 | } 35 | } else { 36 | copyOnWriteTreeSet.add(Integer.parseInt(pair)); 37 | } 38 | } 39 | return copyOnWriteTreeSet; 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/cn/iinti/proxycompose/utils/ResourceUtil.java: -------------------------------------------------------------------------------- 1 | package cn.iinti.proxycompose.utils; 2 | 3 | import org.apache.commons.io.IOUtils; 4 | 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.List; 9 | 10 | public class ResourceUtil { 11 | public static List readLines(String resourceName) { 12 | try (InputStream inputStream = openResource(resourceName)) { 13 | return IOUtils.readLines(inputStream, StandardCharsets.UTF_8); 14 | } catch (IOException e) { 15 | throw new RuntimeException(e); 16 | } 17 | } 18 | 19 | public static String readText(String resourceName) { 20 | try (InputStream inputStream = openResource(resourceName)) { 21 | return IOUtils.toString(inputStream, StandardCharsets.UTF_8); 22 | } catch (IOException e) { 23 | throw new RuntimeException(e); 24 | } 25 | } 26 | 27 | public static byte[] readBytes(String resourceName) { 28 | try (InputStream inputStream = openResource(resourceName)) { 29 | return IOUtils.toByteArray(inputStream); 30 | } catch (IOException e) { 31 | throw new RuntimeException(e); 32 | } 33 | } 34 | 35 | public static InputStream openResource(String name) { 36 | InputStream resource = ResourceUtil.class.getClassLoader() 37 | .getResourceAsStream(name); 38 | if (resource == null) { 39 | throw new IllegalStateException("can not find resource: " + name); 40 | } 41 | return resource; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/resources/config.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | auth_username=iinti 3 | auth_password=iinti 4 | 5 | [source:dailiyun] 6 | loadURL=http://修改这里.user.xiecaiyun.com/api/proxies?action=getText&key=修改这里&count=修改这里&word=&rand=false&norepeat=false&detail=false<ime=0 7 | upstreamAuthUser=修改这里 8 | upstreamAuthPassword=修改这里 9 | poolSize=10 -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | 15 | 16 | 17 | ${LOG_DIR}/${LOG_PROJECT_NAME}/${LOG_PROJECT_NAME}-service.log 18 | 19 | ${normal-pattern} 20 | 21 | 22 | ${LOG_DIR}/${LOG_PROJECT_NAME}/service/service-%d{yyyy-MM-dd}.zip 23 | 24 | 15 25 | 26 | 27 | 28 | 29 | 30 | Scene 31 | unknown 32 | 33 | 34 | 35 | ${LOG_DIR}/${LOG_PROJECT_NAME}/traces/${Scene}/event.log 36 | 37 | 38 | [%d] %m%n 39 | 40 | 41 | ${LOG_DIR}/${LOG_PROJECT_NAME}/traces/${Scene}/event-%d{yyyy-MM-dd}.zip 42 | 43 | ${maxHistory} 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | ERROR 52 | ACCEPT 53 | DENY 54 | 55 | ${LOG_DIR}/${LOG_PROJECT_NAME}/${LOG_PROJECT_NAME}-error.log 56 | 57 | ${LOG_DIR}/${LOG_PROJECT_NAME}/errors/error-%d{yyyy-MM-dd}.zip 58 | ${errorMaxHistory} 59 | 60 | 61 | ${normal-pattern} 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | --------------------------------------------------------------------------------