├── .gitignore ├── LICENSE ├── README.md ├── build.gradle.kts ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle.kts └── src ├── main ├── kotlin │ └── pe │ │ └── proxy │ │ └── proxybuilder2 │ │ ├── ProxyBuilder2Application.kt │ │ ├── caching │ │ └── Caching.kt │ │ ├── controller │ │ ├── ErrorController.kt │ │ ├── HomeController.kt │ │ └── ProxyController.kt │ │ ├── database │ │ ├── DatabaseData.kt │ │ ├── ProxyEntity.kt │ │ ├── ProxyInteraction.kt │ │ └── ProxyRepository.kt │ │ ├── git │ │ ├── GitActions.kt │ │ └── Shell.kt │ │ ├── monitor │ │ ├── CachingMonitor.kt │ │ ├── ChecksMonitor.kt │ │ ├── EndpointMonitor.kt │ │ └── SQLProxyMonitor.kt │ │ ├── net │ │ └── proxy │ │ │ ├── data │ │ │ └── ProxyData.kt │ │ │ ├── proxycheckio │ │ │ ├── QueryApi.kt │ │ │ └── QueryData.kt │ │ │ ├── supplier │ │ │ ├── CustomProxySupplier.kt │ │ │ ├── DatabaseProxySupplier.kt │ │ │ ├── IProxySupplier.kt │ │ │ ├── LocalProxySupplier.kt │ │ │ └── MainProxySupplier.kt │ │ │ └── tester │ │ │ ├── ProxyChannelData.kt │ │ │ ├── ProxyChannelEncoderDecoder.kt │ │ │ ├── ProxyChannelHandler.kt │ │ │ ├── ProxyChannelInitializer.kt │ │ │ └── ProxyConnect.kt │ │ ├── security │ │ └── SecurityConfig.kt │ │ └── util │ │ ├── CustomCacheManager.kt │ │ ├── KotlinDeserializer.kt │ │ ├── KotlinSerializer.kt │ │ ├── ProxyConfig.kt │ │ ├── Tasks.kt │ │ ├── Utils.kt │ │ └── writer │ │ ├── CustomFileWriter.kt │ │ └── ReadMeFile.kt └── resources │ └── application.properties └── test └── kotlin └── pe └── proxy └── proxybuilder2 ├── MainProxySupplierTest.kt ├── ProxyBuilder2ApplicationTests.kt └── SerializerTests.kt /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | .gradle 3 | build/ 4 | !gradle/wrapper/gradle-wrapper.jar 5 | !**/src/main/**/build/ 6 | !**/src/test/**/build/ 7 | 8 | ### Resources ### 9 | **/src/main/resources/** 10 | *.yml 11 | *.jks 12 | *.pfx 13 | *.p12 14 | proxies/ 15 | 16 | ### STS ### 17 | .apt_generated 18 | .classpath 19 | .factorypath 20 | .project 21 | .settings 22 | .springBeans 23 | .sts4-cache 24 | bin/ 25 | !**/src/main/**/bin/ 26 | !**/src/test/**/bin/ 27 | 28 | ### IntelliJ IDEA ### 29 | .idea 30 | *.iws 31 | *.iml 32 | *.ipr 33 | out/ 34 | !**/src/main/**/out/ 35 | !**/src/test/**/out/ 36 | 37 | ### NetBeans ### 38 | /nbproject/private/ 39 | /nbbuild/ 40 | /dist/ 41 | /nbdist/ 42 | /.nb-gradle/ 43 | 44 | ### VS Code ### 45 | .vscode/ 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Kai o((>ω< ))o 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ###### Proxy Builder 2.0 2 | 3 | ## Requirements: 4 | 5 | - ✅ IntelliJ IDEA 6 | - ✅ JDK 18 7 | - ✅ Kotlin 1.8.0 8 | - ✅ Internet Connection 9 | - ✅ MariaDB (or another SQL alternative) 10 | - ✅ 100MB Storage 11 | 12 | ## New features currently include: 13 | - Location data 14 | - Country 15 | - Continent 16 | - (Etc...) 17 | - Provider data 18 | - ISP/Hosting provider 19 | - Connection endpoint information 20 | - (How many times it had connceted to XYZ endpoint) 21 | - Ping 22 | - Uptime 23 | - Merged proxies 24 | - Combining SOCKS4, SOCKS5, HTTP & HTTPS together along with the ports that match the protocol 25 | 26 | View Types Available: 27 | - Classic 28 | - Current view (http, https, socks4, socks5) as ip:port in an array 29 | - Basic 30 | - Mapped data (ip -> "0.0.0.0", port -> 12345, ping -> 120) etc... 31 | - Advanced 32 | - Mapped data, same format as basic but includes Location data + Provider Data etc... 33 | 34 | ```json 35 | [{ 36 | "ip" : "1.10.141.220", 37 | "port" : 54620, 38 | "ping" : 739, 39 | "protocols" : [ { 40 | "type" : "https", 41 | "port" : 54620 42 | }, { 43 | "type" : "http", 44 | "port" : 54620 45 | }, { 46 | "type" : "socks4", 47 | "port" : 37718 48 | } ] 49 | }] 50 | ``` 51 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 2 | 3 | plugins { 4 | id("org.springframework.boot") version "3.0.2" 5 | id("io.spring.dependency-management") version "1.1.0" 6 | //id("org.springframework.experimental.aot") version "0.11.5" //(LOGGING BUG) 7 | 8 | kotlin("jvm") version "1.8.10" 9 | kotlin("plugin.spring") version "1.8.10" 10 | kotlin("plugin.serialization") version "1.8.10" 11 | } 12 | 13 | group = "pe.proxy" 14 | version = "0.0.1-SNAPSHOT" 15 | java.sourceCompatibility = JavaVersion.VERSION_18 16 | 17 | configurations { 18 | compileOnly { 19 | extendsFrom(configurations.annotationProcessor.get()) 20 | } 21 | } 22 | 23 | tasks.register("runDev") { 24 | group = "application" 25 | description = "Runs the Spring Boot application with the dev profile" 26 | doFirst { 27 | tasks.bootRun.configure { 28 | systemProperty("spring.profiles.active", "dev") 29 | } 30 | } 31 | finalizedBy("bootRun") 32 | } 33 | 34 | repositories { 35 | mavenCentral() 36 | //maven { url = uri("https://repo.spring.io/release") } 37 | //maven { url = uri("https://repo.spring.io/milestone") } 38 | //maven { url = uri("https://repo.spring.io/snapshot") } 39 | } 40 | 41 | dependencies { 42 | //Jackson Modules for serialization/deserialization 43 | implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2") 44 | implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.2") 45 | implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.2") 46 | implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.14.2") 47 | implementation("com.fasterxml.jackson.dataformat:jackson-dataformats-text:2.14.2") 48 | 49 | //Apache CSV 50 | implementation("org.apache.commons:commons-csv:1.10.0") 51 | 52 | //Netty4 - Connecting to Endpoint Test Server, Testing the Proxy 53 | implementation("io.netty:netty-handler-proxy:4.1.87.Final") 54 | //implementation("io.netty:netty-all:4.1.78.Final") 55 | 56 | //JPA API - Interacting with JDBC 57 | implementation("org.springframework.boot:spring-boot-starter-data-jpa") 58 | 59 | //Spring Web - Minor API for checking if Application is running 60 | implementation("org.springframework.boot:spring-boot-starter-web") 61 | 62 | //Twilio API (Sending alerts if detects one of the endpoint servers are down) 63 | //TODO - Switch this to just a simple REST request, library isn't needed for EndpointMonitor 64 | implementation("com.twilio.sdk:twilio:9.2.3") 65 | 66 | //Caching 67 | implementation("com.github.ben-manes.caffeine:caffeine:3.1.3") 68 | 69 | //WebFlux & Reactor - Minor API for checking if Application is running 70 | //implementation("org.springframework.boot:spring-boot-starter-webflux") 71 | //implementation("io.projectreactor.kotlin:reactor-kotlin-extensions") 72 | 73 | //Spring Security - Allow access to specific directory's 74 | implementation("org.springframework.boot:spring-boot-starter-security") 75 | 76 | //Kotlin Defaults 77 | implementation("org.jetbrains.kotlin:kotlin-reflect") 78 | implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") 79 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactor:1.6.4") 80 | //implementation("org.junit.jupiter:junit-jupiter:5.8.2") 81 | 82 | //KotlinX - Serializing JSON data to Kotlin Class 83 | implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.5.0-RC") 84 | 85 | //Jsoup to sanitize html 86 | //implementation("org.jsoup:jsoup:1.15.1") 87 | 88 | //Unit Testing 89 | testImplementation("org.springframework.boot:spring-boot-starter-test") 90 | testImplementation("io.projectreactor:reactor-test:3.4.24") 91 | 92 | //testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.2") 93 | //testImplementation("org.mockito:mockito-core:4.5.1") 94 | //testImplementation("org.hamcrest:hamcrest:2.2") 95 | 96 | //Configuration Processor - For YAML configs 97 | annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") 98 | 99 | //Devtools - Live debugging 100 | developmentOnly("org.springframework.boot:spring-boot-devtools") 101 | 102 | //MariaDB - SQL Database for storing the tested Proxies 103 | runtimeOnly("org.mariadb.jdbc:mariadb-java-client:3.1.2") 104 | //Reflection for Kotlin 105 | runtimeOnly("org.jetbrains.kotlin:kotlin-reflect:1.8.10") 106 | } 107 | 108 | tasks.withType { 109 | kotlinOptions { 110 | freeCompilerArgs = listOf("-Xjsr305=strict") 111 | jvmTarget = "18" 112 | apiVersion = "1.8" 113 | languageVersion = "1.8" 114 | } 115 | } 116 | 117 | tasks.withType { 118 | useJUnitPlatform() 119 | } 120 | 121 | /* 122 | tasks.withType { 123 | builder = "paketobuildpacks/builder:tiny" 124 | environment = mapOf("BP_NATIVE_IMAGE" to "true") 125 | }*/ 126 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jetkai/proxy-builder-2/4ab37090dafa87d63afd9a4d1e25583a78bcffe0/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-rc-5-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | maven { url = uri("https://repo.spring.io/release") } 4 | gradlePluginPortal() 5 | } 6 | } 7 | rootProject.name = "proxy-builder-2" 8 | -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/ProxyBuilder2Application.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2 2 | 3 | import org.springframework.boot.autoconfigure.SpringBootApplication 4 | import org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration 5 | import org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration 6 | import org.springframework.boot.context.properties.EnableConfigurationProperties 7 | import org.springframework.boot.runApplication 8 | import org.springframework.cache.annotation.EnableCaching 9 | import pe.proxy.proxybuilder2.util.ProxyConfig 10 | 11 | /** 12 | * ProxyBuilder2Application 13 | * 14 | * @author Kai 15 | * @version 1.0, 19/05/2022 16 | */ 17 | 18 | @EnableCaching 19 | @SpringBootApplication(exclude = [(UserDetailsServiceAutoConfiguration::class), (ErrorMvcAutoConfiguration::class)]) 20 | @EnableConfigurationProperties(ProxyConfig::class) 21 | class ProxyBuilder2Application 22 | 23 | fun main(args: Array) { 24 | runApplication(*args) 25 | } 26 | -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/caching/Caching.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.caching 2 | 3 | import com.github.benmanes.caffeine.cache.Caffeine 4 | import java.util.concurrent.TimeUnit 5 | 6 | //TODO Caffeine Caching - Currently using Spring Caching 7 | class Caching { 8 | 9 | //@Bean 10 | fun config(): Caffeine { 11 | return Caffeine.newBuilder().expireAfterWrite(15, TimeUnit.SECONDS) 12 | } 13 | 14 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/controller/ErrorController.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.controller 2 | 3 | //@ControllerAdvice 4 | class ErrorController { 5 | 6 | /* @ExceptionHandler(value = [(IllegalArgumentException::class)]) 7 | private fun onErrorResponse(exception : IllegalArgumentException) : ResponseEntity = 8 | ResponseEntity(exception.message?.let { Jsoup.parse(it).text() }, HttpStatus.INTERNAL_SERVER_ERROR)*/ 9 | 10 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/controller/HomeController.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.controller 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude 4 | import com.fasterxml.jackson.databind.annotation.JsonSerialize 5 | import jakarta.servlet.http.Cookie 6 | import jakarta.servlet.http.HttpServletRequest 7 | import org.springframework.http.HttpStatus 8 | import org.springframework.http.ResponseEntity 9 | import org.springframework.web.bind.annotation.RequestMapping 10 | import org.springframework.web.bind.annotation.RequestMethod 11 | import org.springframework.web.bind.annotation.RestController 12 | import java.util.* 13 | 14 | @RestController 15 | class HomeController { 16 | 17 | @RequestMapping(value = ["/"], produces = ["application/json"], method = [RequestMethod.GET]) 18 | private fun onHomeResponse(request : HttpServletRequest) : ResponseEntity { 19 | val entity = HomeResponseEntity(request) 20 | return ResponseEntity(entity, HttpStatus.OK) 21 | } 22 | 23 | } 24 | 25 | /** 26 | * HomeResponseEntity 27 | * 28 | * @author Kai 29 | * @version 1.0, 12/06/2022 30 | */ 31 | @JsonSerialize 32 | @JsonInclude(JsonInclude.Include.NON_NULL) //Ignore nulls 33 | @Suppress("unused") //All val's being used 34 | internal class HomeResponseEntity(private val request : HttpServletRequest) { 35 | 36 | val remoteAddr : String = request.remoteAddr 37 | val protocol : String = request.protocol 38 | val contentLength = request.contentLength 39 | val contentType : String = request.contentType 40 | val cookies : Array = request.cookies 41 | val queryString : String = request.queryString 42 | val locale : Locale = request.locale 43 | val timestamp = Date().time 44 | val headers : Map 45 | get() { 46 | val map : MutableMap = HashMap() 47 | val headers = request.headerNames.asIterator() 48 | while (headers.hasNext()) { 49 | val headerName = headers.next() 50 | map[headerName] = request.getHeader(headerName) 51 | } 52 | return map 53 | } 54 | 55 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/controller/ProxyController.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.controller 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude 4 | import com.fasterxml.jackson.core.JsonFactory 5 | import com.fasterxml.jackson.databind.ObjectMapper 6 | import com.fasterxml.jackson.databind.SerializationFeature 7 | import com.fasterxml.jackson.dataformat.csv.CsvFactory 8 | import com.fasterxml.jackson.dataformat.csv.CsvMapper 9 | import com.fasterxml.jackson.dataformat.csv.CsvSchema 10 | import com.fasterxml.jackson.dataformat.xml.XmlFactory 11 | import com.fasterxml.jackson.dataformat.xml.XmlMapper 12 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 13 | import org.slf4j.LoggerFactory 14 | import org.springframework.cache.CacheManager 15 | import org.springframework.http.HttpStatus 16 | import org.springframework.http.ResponseEntity 17 | import org.springframework.web.bind.annotation.RequestMapping 18 | import org.springframework.web.bind.annotation.RequestMethod 19 | import org.springframework.web.bind.annotation.RequestParam 20 | import org.springframework.web.bind.annotation.RestController 21 | import pe.proxy.proxybuilder2.database.EntityForPublicView 22 | import pe.proxy.proxybuilder2.database.EntityForPublicViewForCSV 23 | import pe.proxy.proxybuilder2.database.ProxyRepository 24 | import pe.proxy.proxybuilder2.net.proxy.data.SupplierProxyListData 25 | import pe.proxy.proxybuilder2.util.CustomCacheManager 26 | import pe.proxy.proxybuilder2.util.Utils 27 | import pe.proxy.proxybuilder2.util.writer.CustomFileWriter 28 | 29 | /** 30 | * ProxyController Class 31 | * 32 | * Used as a RestController for API Queries 33 | * @see onRunningCheck 34 | * 35 | * @author Kai 36 | * @version 1.0, 15/05/2022 37 | */ 38 | @RestController 39 | @RequestMapping("/api/v1/") 40 | class ProxyController(private val repository : ProxyRepository, cacheManager : CacheManager) { 41 | 42 | private val logger = LoggerFactory.getLogger(ProxyController::class.java) 43 | 44 | private val cache = CustomCacheManager(cacheManager) 45 | 46 | /** 47 | * Running Check 48 | * @since version 1.0 49 | * @return This function will return a JSON response with the current running data 50 | */ 51 | @RequestMapping(value = ["running"], produces = ["application/json"], method = [RequestMethod.GET]) 52 | private fun onRunningCheck() : ResponseEntity = ResponseEntity("HELLO", HttpStatus.OK) 53 | 54 | @Throws 55 | @RequestMapping(value = ["proxies"], produces = ["application/json"], method = [RequestMethod.GET]) 56 | private fun onRequestForOnlineProxies( 57 | @RequestParam(value = "protocols", required = false, defaultValue = "any") protocols : List, 58 | @RequestParam(value = "countries", required = false, defaultValue = "any") countries : List, 59 | @RequestParam(value = "format", required = false, defaultValue = "txt") format : String, 60 | @RequestParam(value = "amount", required = false, defaultValue = "0") amount : Int, 61 | @RequestParam(value = "stable", required = false, defaultValue = "false") stable : Boolean, 62 | @RequestParam(value = "type", required = false, defaultValue = "basic") type : String 63 | ) : ResponseEntity { 64 | 65 | //Grab the latest list of proxies 66 | //Clone list so that we do not modify the existing list in the cache 67 | var cachedProxies : MutableList = getProxiesFromCache().toMutableList() 68 | 69 | val allowedTypes = arrayOf("classic", "basic", "advanced") 70 | if(!allowedTypes.contains(type)) 71 | return ResponseEntity("Bad type.", HttpStatus.OK) 72 | 73 | //HANDLE -> Basic & Advanced Format (MutableList) 74 | if(cachedProxies.isNotEmpty()) { 75 | 76 | //Only get protocols specified 77 | if (protocols.isNotEmpty() && !protocols.contains("any")) { 78 | cachedProxies = protocols.flatMap { prot -> 79 | cachedProxies.filter { prox -> 80 | prox.protocols?.any { it.type == prot } == true 81 | } 82 | }.toMutableList() 83 | } 84 | 85 | //Filter only the countries the user has specified 86 | if (countries.isNotEmpty() && !countries.contains("any")) 87 | cachedProxies = countries.flatMap { country -> 88 | cachedProxies.filter { prox -> 89 | if (country.length == 2) //ISOCode is 2 chars long 90 | prox.location?.isocode?.lowercase() == country.lowercase() 91 | else 92 | prox.location?.country?.lowercase() == country.lowercase() 93 | } 94 | }.toMutableList() 95 | 96 | //Shuffle them up, so we do not provide in ip-order 97 | cachedProxies.shuffle() 98 | 99 | //Sort by most stable proxies (by highest uptime) 100 | if (stable) 101 | cachedProxies = cachedProxies.sortedBy { it.endpoints?.ora_UK?.uptime?.replace("%", "")?.toDouble() } 102 | .asReversed().toMutableList() 103 | 104 | //Only provide the max amount of proxies the user requests 105 | var maxAmount = amount 106 | if (maxAmount > cachedProxies.size) 107 | maxAmount = cachedProxies.size 108 | if (maxAmount > 0) 109 | cachedProxies = cachedProxies.subList(0, maxAmount) 110 | 111 | //Reduce fields to be more basic 112 | if(type == "basic") { 113 | cachedProxies.forEach { 114 | it.ping = it.endpoints?.let { endpoint -> Utils.lowestPing(endpoint) } 115 | it.credentials = null 116 | it.endpoints = null 117 | it.dateAdded = null 118 | it.lastSuccess = null 119 | it.lastTested = null 120 | it.location = null 121 | it.detection = null 122 | it.provider = null 123 | it.protocols?.forEach { prot -> 124 | prot.autoRead = null 125 | prot.tls = null 126 | } 127 | } 128 | } 129 | 130 | //Finally, produce the finished output 131 | //Do not allow classic view or TXT pass through this format, more stuff needs to be done for classic & txt 132 | if(type != "classic" && format.uppercase() != "TXT") { 133 | val responseBody = formatEntityListToString(cachedProxies, format.uppercase()) 134 | return ResponseEntity(responseBody, HttpStatus.OK) 135 | } 136 | } 137 | 138 | //HANDLE -> Classic Format (SupplierProxyListData) & TXT Format 139 | //We will also be working off the Advanced Cached Data, as Classic Format does not have location/speed info 140 | if(!cachedProxies.isNullOrEmpty()) { 141 | 142 | //Fresh list to build 143 | val classicProxies = SupplierProxyListData( 144 | mutableListOf(), mutableListOf(), mutableListOf(), mutableListOf() 145 | ) 146 | 147 | //Linking protocols to a string, so it's easier to loop through the lists 148 | val linkProtocols = arrayOf( 149 | arrayOf("http", classicProxies.http), 150 | arrayOf("https", classicProxies.https), 151 | arrayOf("socks4", classicProxies.socks4), 152 | arrayOf("socks5", classicProxies.socks5) 153 | ) 154 | 155 | for(proxy in cachedProxies) { 156 | proxy.protocols?.forEach { proxyProtocol -> 157 | val proxyList = linkProtocols.filter { it[0] == proxyProtocol.type }[0][1] as? MutableList 158 | proxyList?.add("${proxy.ip}:${proxyProtocol.port}") 159 | } 160 | } 161 | 162 | for(linkProtocol in linkProtocols) { 163 | val protocol = linkProtocol[0] 164 | val proxyList = linkProtocol[1] as? MutableList 165 | if(!protocols.contains(protocol)) 166 | proxyList?.clear() 167 | } 168 | 169 | val amountToRemove = classicProxies.size() - amount 170 | classicProxies.removeAtRandom(amountToRemove) 171 | 172 | //Finally, produce the finished output 173 | val responseBody = formatEntityListToString(classicProxies, format.uppercase()) 174 | return ResponseEntity(responseBody, HttpStatus.OK) 175 | } 176 | 177 | //TODO - Add database to track if this is being used 178 | logger.info("Produced response") 179 | 180 | return ResponseEntity("Empty.", HttpStatus.NO_CONTENT) 181 | } 182 | 183 | //Returns the file format requested (as a string) 184 | //THIS IS FOR ADVANCED VIEW 185 | private fun formatEntityListToString(proxies: List, format: String) : String { 186 | val extension = try { CustomFileWriter.FileExtension.valueOf(format) 187 | } catch (e : IllegalArgumentException) { 188 | logger.error(e.message) 189 | CustomFileWriter.FileExtension.valueOf("TXT") 190 | } 191 | 192 | val mapper = getMapper(extension) 193 | ?.setSerializationInclusion(JsonInclude.Include.NON_NULL) 194 | ?.enable(SerializationFeature.INDENT_OUTPUT) 195 | 196 | if(extension == CustomFileWriter.FileExtension.CSV && mapper is CsvMapper) { 197 | val entity = EntityForPublicViewForCSV().convert(proxies) 198 | val schema: CsvSchema = mapper.schemaFor(EntityForPublicViewForCSV::class.java) 199 | .withHeader().sortedBy(* EntityForPublicViewForCSV().order(CustomFileWriter.ViewType.ADVANCED)) 200 | return mapper.writerFor(List::class.java).with(schema).writeValueAsString(entity) 201 | } 202 | 203 | return mapper?.writeValueAsString(proxies) ?: "" 204 | } 205 | 206 | //Returns the file format requested (as a string) 207 | //THIS IS FOR CLASSIC VIEW 208 | private fun formatEntityListToString(proxies : SupplierProxyListData, format : String) : String { 209 | val extension = try { CustomFileWriter.FileExtension.valueOf(format) 210 | } catch (e : IllegalArgumentException) { 211 | logger.error(e.message) 212 | CustomFileWriter.FileExtension.valueOf("TXT") 213 | } 214 | 215 | if(extension == CustomFileWriter.FileExtension.TXT) { 216 | val allProxies = (proxies.http + proxies.https + proxies.socks4 + proxies.socks5).shuffled() 217 | return allProxies.joinToString(separator = "\n") { it } 218 | } 219 | 220 | val mapper = getMapper(extension) 221 | ?.setSerializationInclusion(JsonInclude.Include.NON_NULL) 222 | ?.enable(SerializationFeature.INDENT_OUTPUT) 223 | 224 | if(extension == CustomFileWriter.FileExtension.CSV && mapper is CsvMapper) 225 | return "TODO" 226 | 227 | return mapper?.writeValueAsString(proxies) ?: "" 228 | } 229 | 230 | private fun getProxiesFromCache() : MutableList { 231 | val cachedProxies = cache.getCachedProxies() 232 | 233 | //Return if proxies from cache, if they are already cached 234 | if(!cachedProxies.isNullOrEmpty()) 235 | return cachedProxies 236 | 237 | val lastOnlineSince = Utils.timestampMinus(9000) //Within the past 90 minutes 238 | val repoResults = repository.findByLastSuccessAfter(lastOnlineSince) 239 | 240 | val proxies = mutableListOf() 241 | 242 | //Map the objects to mutableLists 243 | repoResults.mapTo(proxies) { EntityForPublicView().advanced(it) } 244 | 245 | //Add to cache if doesn't already exist in cache 246 | cache.getCache().getCache("ProxyController_proxies")?.put("advanced", proxies) 247 | 248 | return proxies 249 | } 250 | 251 | private fun getMapper(extension : CustomFileWriter.FileExtension) : ObjectMapper? { 252 | return when (extension) { 253 | CustomFileWriter.FileExtension.YAML -> { ObjectMapper(YAMLFactory()) } 254 | CustomFileWriter.FileExtension.JSON -> { ObjectMapper(JsonFactory()) } 255 | CustomFileWriter.FileExtension.XML -> { XmlMapper(XmlFactory()) } 256 | CustomFileWriter.FileExtension.CSV -> { CsvMapper(CsvFactory()) } 257 | else -> null 258 | } 259 | } 260 | 261 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/database/DatabaseData.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.database 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore 4 | import com.fasterxml.jackson.annotation.JsonProperty 5 | import kotlinx.serialization.Serializable 6 | import pe.proxy.proxybuilder2.net.proxy.data.* 7 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.LocationData 8 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.OperatorData 9 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.RiskData 10 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelData 11 | import pe.proxy.proxybuilder2.util.KotlinDeserializer 12 | import pe.proxy.proxybuilder2.util.Utils 13 | import pe.proxy.proxybuilder2.util.writer.CustomFileWriter 14 | 15 | /** 16 | * DatabaseData 17 | * 18 | * @author Kai 19 | * @version 1.0, 23/05/2022 20 | */ 21 | data class EntityChannelData(val entity : ProxyEntity, val proxy : ProxyChannelData) 22 | 23 | @Serializable 24 | data class EntityForPublicView( 25 | var ip : String? = null, 26 | var port : Int? = null, 27 | var ping : Long? = null, 28 | var uptime : Int? = null, 29 | var protocols : List? = null, 30 | var credentials : ProxyCredentials? = null, 31 | var endpoints : PerformanceConnectData? = null, 32 | var detection : RiskData? = null, 33 | var provider : OperatorData? = null, 34 | var location : LocationData? = null, 35 | @JsonProperty("date_added") 36 | var dateAdded : String? = null, 37 | @JsonProperty("last_tested") 38 | var lastTested : String? = null, 39 | @JsonProperty("last_success") 40 | var lastSuccess : String? = null) { 41 | 42 | @JsonIgnore 43 | fun classic(entity : SupplierProxyListData, proxy : ProxyEntity) : SupplierProxyListData { 44 | if(proxy.protocols?.contains("\"http\"") == true) 45 | entity.http.add("${proxy.ip}:${proxy.port}") 46 | if(proxy.protocols?.contains("\"https\"") == true) 47 | entity.https.add("${proxy.ip}:${proxy.port}") 48 | if(proxy.protocols?.contains("\"socks4\"") == true) 49 | entity.socks4.add("${proxy.ip}:${proxy.port}") 50 | if(proxy.protocols?.contains("\"socks5\"") == true) 51 | entity.socks5.add("${proxy.ip}:${proxy.port}") 52 | return entity 53 | } 54 | 55 | //Temp deserializer (for testing) - this is currently hybrid with KTX & Jackson - Bad 56 | //Jackson doesn't parse KTX string decode properly, not sure how to parse KTX JsonElement to Jackson ATM 57 | @JsonIgnore 58 | fun minimal(proxy : ProxyEntity) : EntityForPublicView { 59 | ip = proxy.ip 60 | port = proxy.port 61 | protocols = KotlinDeserializer.decode(proxy.protocols)?.protocol 62 | return this 63 | } 64 | 65 | //Temp deserializer (for testing) - this is currently hybrid with KTX & Jackson - Bad 66 | //Jackson doesn't parse KTX string decode properly, not sure how to parse KTX JsonElement to Jackson ATM 67 | @JsonIgnore 68 | fun basic(proxy : ProxyEntity) : EntityForPublicView { 69 | ip = proxy.ip 70 | port = proxy.port 71 | protocols = KotlinDeserializer.decode(proxy.protocols)?.protocol 72 | ping = KotlinDeserializer.decode(proxy.connections)?.let { Utils.lowestPing(it) } 73 | return this 74 | } 75 | 76 | //Temp deserializer (for testing) - this is currently hybrid with KTX & Jackson - Bad 77 | //Jackson doesn't parse KTX string decode properly, not sure how to parse KTX JsonElement to Jackson ATM 78 | @JsonIgnore 79 | fun advanced(proxy : ProxyEntity) : EntityForPublicView { 80 | ip = proxy.ip 81 | port = proxy.port 82 | credentials = KotlinDeserializer.decode(proxy.credentials) 83 | protocols = KotlinDeserializer.decode(proxy.protocols)?.protocol 84 | endpoints = KotlinDeserializer.decode(proxy.connections) 85 | detection = KotlinDeserializer.decode(proxy.detection) 86 | provider = try { 87 | KotlinDeserializer.decode(proxy.provider) 88 | } catch (e : Exception) { 89 | OperatorData(proxy.provider) 90 | } 91 | location = KotlinDeserializer.decode(proxy.location) 92 | dateAdded = proxy.dateAdded.toString() 93 | lastSuccess = proxy.lastSuccess.toString() 94 | lastTested = proxy.lastTested.toString() 95 | return this 96 | } 97 | 98 | } 99 | 100 | data class EntityForPublicViewForCSV( 101 | var ip: String? = null, 102 | var port: Int? = null, 103 | var ping: Long? = null, 104 | var protocols: String? = null, 105 | var username: String? = null, 106 | var password: String? = null, 107 | var detected: Boolean? = null, 108 | var provider: String? = null, 109 | var organisation: String? = null, 110 | var country: String? = null, 111 | var isocode: String? = null, 112 | var latitude: Float? = null, 113 | var longitude: Float? = null, 114 | var asn: String? = null, 115 | var connections: String? = null, 116 | var detection: String? = null, 117 | var uptime: Int? = null, 118 | @JsonProperty("date_added") 119 | var dateAdded: String? = null, 120 | @JsonProperty("last_tested") 121 | var lastTested: String? = null, 122 | @JsonProperty("last_success") 123 | var lastSuccess: String? = null) { 124 | 125 | //TODO - Change this, creating extra EntityForPublicViewForCSV() for no reason 126 | @JsonIgnore 127 | fun convert(entities : List) : List { 128 | val entitiesCsv = mutableListOf() 129 | for(entity in entities) { 130 | val entityCsv = EntityForPublicViewForCSV() 131 | entityCsv.ip = entity.ip 132 | entityCsv.port = entity.port 133 | entityCsv.ping = entity.ping 134 | entityCsv.protocols = entity.protocols.toString() 135 | entityCsv.username = entity.credentials?.username 136 | entityCsv.password = entity.credentials?.password 137 | entityCsv.detected = entity.detection?.proxy 138 | entityCsv.organisation = entity.location?.organisation 139 | entityCsv.country = entity.location?.country 140 | entityCsv.isocode = entity.location?.isocode 141 | entityCsv.latitude = entity.location?.latitude 142 | entityCsv.longitude = entity.location?.longitude 143 | entityCsv.asn = entity.location?.asn 144 | entityCsv.connections = entity.endpoints.toString() 145 | entityCsv.uptime = entity.uptime 146 | entityCsv.dateAdded = entity.dateAdded 147 | entityCsv.lastTested = entity.lastTested 148 | entityCsv.dateAdded = entity.lastSuccess 149 | entitiesCsv.add(entityCsv) 150 | } 151 | return entitiesCsv 152 | } 153 | 154 | @JsonIgnore 155 | fun order(viewType : CustomFileWriter.ViewType): Array { 156 | return when (viewType) { 157 | CustomFileWriter.ViewType.ADVANCED, CustomFileWriter.ViewType.ARCHIVE -> { 158 | arrayOf( 159 | "ip", "port", "username", "password", "protocols", "ping", "detected", 160 | "organisation", "country", "isocode", "latitude", "longitude", "asn", 161 | "connections", "uptime", "dateAdded", "lastTested", "lastSuccess" 162 | ) 163 | } 164 | CustomFileWriter.ViewType.BASIC -> { arrayOf("ip", "port", "protocols", "ping") } 165 | else -> { arrayOf("http", "https", "socks4", "socks5") } 166 | } 167 | } 168 | 169 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/database/ProxyEntity.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.database 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore 4 | import com.fasterxml.jackson.annotation.JsonInclude 5 | import jakarta.persistence.* 6 | import java.sql.Timestamp 7 | 8 | /** 9 | * ProxyEntity 10 | * 11 | * Serializer/Deserializer placeholder 12 | * Reserve Name Ref: https://dev.mysql.com/doc/refman/8.0/en/keywords.html#keywords-8-0-detailed-I 13 | * (Future revision use OneToOne https://www.baeldung.com/jpa-one-to-one & deprecate json) 14 | * https://blog.ippon.tech/boost-the-performance-of-your-spring-data-jpa-application/ 15 | * 16 | * @author Kai 17 | * @version 1.0, 15/05/2022 18 | */ 19 | @Entity 20 | @Table(name = "all_proxies", schema = "localdb") 21 | @JsonInclude(JsonInclude.Include.NON_NULL) 22 | class ProxyEntity { 23 | 24 | @Id 25 | @GeneratedValue(strategy=GenerationType.IDENTITY) 26 | @Column(name = "id", nullable = false) 27 | @JsonIgnore 28 | var id: Int? = null 29 | 30 | @Basic 31 | @Column(name = "ip", nullable = false) 32 | var ip: String? = null 33 | 34 | @Basic 35 | @Column(name = "port", nullable = false) 36 | var port : Int? = null 37 | 38 | // @Enumerated(EnumType.STRING) 39 | // @JoinColumn(name = "protocols") 40 | @Basic 41 | @Column(name = "protocols", nullable = true) 42 | var protocols : String ?=null 43 | 44 | @Basic 45 | @Column(name = "credentials", nullable = true) 46 | var credentials : String? = null 47 | 48 | @Basic 49 | @Column(name = "location", nullable = true) 50 | var location: String? = null 51 | 52 | @Basic 53 | @Column(name = "connections", nullable = false) 54 | var connections: String? = null 55 | 56 | @Basic 57 | @Column(name = "detection", nullable = true) 58 | var detection: String? = null 59 | 60 | @Basic 61 | @Column(name = "provider", nullable = true) 62 | var provider : String? = null 63 | 64 | @Basic 65 | @Column(name = "date_added", nullable = false) 66 | var dateAdded: Timestamp? = null 67 | 68 | @Basic 69 | @Column(name = "last_tested", nullable = false) 70 | var lastTested: Timestamp? = null 71 | 72 | @Basic 73 | @Column(name = "last_success", nullable = true) 74 | var lastSuccess: Timestamp? = null 75 | 76 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/database/ProxyInteraction.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.database 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.dao.EmptyResultDataAccessException 5 | import pe.proxy.proxybuilder2.net.proxy.data.* 6 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelData 7 | import pe.proxy.proxybuilder2.util.KotlinDeserializer 8 | import pe.proxy.proxybuilder2.util.KotlinSerializer 9 | import pe.proxy.proxybuilder2.util.Utils 10 | import java.sql.SQLSyntaxErrorException 11 | import java.util.concurrent.ConcurrentLinkedQueue 12 | 13 | /** 14 | * ProxyInteraction 15 | * 16 | * Communicates with MariaDB through CrudRepository interface 17 | * @see updateEntity 18 | * @see getProxyEntity 19 | * @see getDefaultTemplate 20 | * 21 | * @author Kai 22 | * @version 1.0, 15/05/2022 23 | */ 24 | class ProxyInteraction(private val repository : ProxyRepository) { 25 | 26 | private val logger = LoggerFactory.getLogger(ProxyInteraction::class.java) 27 | 28 | //Update single ProxyEntity 29 | fun updateEntity(entity : ProxyEntity, proxy : ProxyChannelData) { 30 | entity.connections = connections(entity, proxy) 31 | entity.credentials = credentials(proxy) 32 | entity.protocols = protocols(entity, proxy, true) 33 | time(entity, proxy) 34 | repository.save(entity) //Writes to DB 35 | } 36 | 37 | fun updateEntities(entityData : List) { 38 | val entities = mutableListOf() 39 | for(data in entityData) { 40 | try { 41 | val protocols = protocols(data.entity, data.proxy, false) 42 | 43 | data.entity.connections = connections(data.entity, data.proxy) 44 | data.entity.credentials = credentials(data.proxy) 45 | data.entity.protocols = protocols(data.entity, data.proxy, true) 46 | time(data.entity, data.proxy) 47 | 48 | if (data.entity.protocols == protocols || data.proxy.response.connected != false) { 49 | entities.add(data.entity) 50 | } else { 51 | //TODO -> New database for proxies that fail to connect, flag dead proxies 52 | } 53 | } catch (e : Exception) { 54 | logger.error(e.localizedMessage) 55 | } 56 | } 57 | repository.saveAll(entities) 58 | } 59 | 60 | //Returns a single instance of ProxyEntity 61 | fun getProxyEntity(proxy : ProxyChannelData) : ProxyEntity? { 62 | return try { //Checks if exists in DB 63 | repository.findByIp(proxy.ip) 64 | } catch (e : EmptyResultDataAccessException) { //Returns the default template if player doesn't exist in DB 65 | if(proxy.response.connected == true) 66 | getDefaultTemplate(proxy.ip, proxy.port) 67 | else null 68 | } catch (sql : SQLSyntaxErrorException) { //This is bad, should not get here! 69 | logger.error(sql.message) 70 | if(proxy.response.connected == true) 71 | getDefaultTemplate(proxy.ip, proxy.port) 72 | else null 73 | } 74 | } 75 | 76 | //Might be very heavy on resources, will test - May be better to update as a single entity 77 | fun getProxyEntities(proxies : ConcurrentLinkedQueue) : List { 78 | //Make copyOfProxies - otherwise we are out of sync when comparing 79 | val copyOfProxies = proxies.distinctBy { it.ip } 80 | val proxyEntityList = mutableListOf() 81 | 82 | try { 83 | val listIps = copyOfProxies.map { it.ip }.distinct() 84 | val repositoryList = repository.findByIpIn(listIps) 85 | 86 | //Existing Proxies (that already exist within the database) 87 | copyOfProxies.flatMap { prox -> 88 | repositoryList 89 | .map { repo -> prox to repo } 90 | .filter { it.second.id != 0 91 | && it.second.ip == it.first.ip } } 92 | .mapTo(proxyEntityList) { EntityChannelData(it.second, it.first) } 93 | 94 | //New Proxies (that do not exist within database) 95 | copyOfProxies.filter { it.ip !in repositoryList.map { repo -> repo.ip } } 96 | //.filter { it.response.connected == true } //Only add proxies that have successfully connected 97 | .mapTo(proxyEntityList) { EntityChannelData(getDefaultTemplate(it.ip, it.port), it) } 98 | 99 | //Keep this as proxies.removeIf and not proxiesCopy, so we can remove the ones we have added to DB 100 | proxies.removeAll(copyOfProxies.toSet()) 101 | //proxies.removeIf { listOf(it.ip, it.type, it.port) in copyOfProxies.map { it2 -> listOf(it2.ip, it2.port, it2.type) } } 102 | } catch (e : Exception) { 103 | logger.error(e.localizedMessage) 104 | } 105 | return proxyEntityList 106 | } 107 | 108 | @Throws 109 | private fun connections(entity : ProxyEntity, proxy : ProxyChannelData) : String { 110 | val connectDataJson = entity.connections 111 | var connectData = KotlinDeserializer.decode(connectDataJson) 112 | if(connectData == null) { 113 | connectData = PerformanceConnectData().default() 114 | } 115 | 116 | var endpointData : EndpointServerData ?= null 117 | 118 | when (proxy.endpointServer?.name) { //Use Reflection in future 119 | "aws_NA" -> { endpointData = connectData.aws_NA } 120 | "ora_UK" -> { endpointData = connectData.ora_UK } 121 | "ora_JP" -> { endpointData = connectData.ora_JP } 122 | "ms_HK" -> { endpointData = connectData.ms_HK } 123 | } 124 | 125 | if(endpointData != null) { 126 | 127 | val connections = endpointData.connections 128 | if(connections != null) { 129 | if (proxy.response.connected == true) 130 | connections.success++ 131 | else 132 | connections.fail++ 133 | 134 | if (connections.success > 0) { 135 | val calculation : Double = 136 | (connections.success * 100 / (connections.success + connections.fail)).toDouble() 137 | endpointData.uptime = "$calculation%" 138 | } 139 | } 140 | 141 | val startTime = proxy.response.startTime 142 | val endTime = proxy.response.endTime 143 | if(startTime != null && endTime != null) 144 | endpointData.ping = (endTime.time - startTime.time) 145 | } 146 | 147 | return KotlinSerializer.encode(connectData) 148 | } 149 | 150 | private fun credentials(proxy: ProxyChannelData) : String? { 151 | val credentialsData = ProxyCredentials(proxy.username, proxy.password) 152 | if(!credentialsData.empty()) 153 | return KotlinSerializer.encode(credentialsData) 154 | 155 | return null 156 | } 157 | 158 | @Throws 159 | private fun protocols(entity : ProxyEntity, proxy : ProxyChannelData, append : Boolean) : String { 160 | val defaultData = mutableListOf( 161 | ProtocolDataType(proxy.type, proxy.port, proxy.response.tls, proxy.response.autoRead) 162 | ) 163 | val protocolsJson = entity.protocols 164 | var protocolData = KotlinDeserializer.decode(protocolsJson) 165 | if (protocolData == null) { 166 | protocolData = ProtocolData(defaultData) 167 | } 168 | 169 | if(append) { 170 | val protocol = protocolData.protocol 171 | val protocolIsNotInList = protocol.none { it.port == proxy.port && it.type == proxy.type } 172 | if (protocolIsNotInList) 173 | protocol.add(ProtocolDataType(proxy.type, proxy.port, proxy.response.tls, proxy.response.autoRead)) 174 | } 175 | 176 | return KotlinSerializer.encode(protocolData) 177 | } 178 | 179 | private fun time(entity : ProxyEntity, proxy : ProxyChannelData) { 180 | val connectedTime = proxy.response.endTime 181 | if(connectedTime != null) 182 | entity.lastSuccess = connectedTime 183 | 184 | entity.lastTested = Utils.timestampNow() 185 | } 186 | 187 | //Returns the default template of the ProxyEntity 188 | private fun getDefaultTemplate(ip: String, port: Int) : ProxyEntity { 189 | val currentTime = Utils.timestampNow() 190 | val entity = ProxyEntity() 191 | entity.id = 0 192 | entity.ip = ip 193 | entity.port = port 194 | entity.protocols = null 195 | entity.credentials = null 196 | entity.connections = null 197 | entity.location = null 198 | entity.detection = null 199 | entity.provider = null 200 | entity.dateAdded = currentTime 201 | entity.lastTested = currentTime 202 | entity.lastSuccess = null 203 | return entity 204 | } 205 | 206 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/database/ProxyRepository.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.database 2 | 3 | import org.springframework.data.repository.CrudRepository 4 | import org.springframework.stereotype.Repository 5 | import java.sql.Timestamp 6 | 7 | /** 8 | * ProxyRepository 9 | * 10 | * Uses CrudRepository interface to run query's on the MariaDB 11 | * 12 | * @author Kai 13 | * @version 1.0, 15/05/2022 14 | */ 15 | @Repository("ProxyRepository") 16 | interface ProxyRepository : CrudRepository { 17 | 18 | fun findByIp(ip : String) : ProxyEntity 19 | 20 | fun findByIpIn(ip : List) : List 21 | 22 | fun findByLocationIsNull() : List 23 | 24 | fun findByLocationIsNullAndLastSuccessIsNotNull() : List 25 | 26 | fun findByLocationIsNotNullAndLastSuccessIsNotNull() : List 27 | 28 | fun findByIpInAndLastSuccessBefore(ip : List, timestamp : Timestamp) : List 29 | fun findByLastSuccessIsNullAndLastTestedBefore(lastTestedBefore : Timestamp): List 30 | 31 | //@Override 32 | //@Cacheable("ProxyRepository_lastSuccess") 33 | fun findByLastSuccessAfter(timestamp : Timestamp) : List 34 | 35 | fun findByLastSuccessIsNotNull() : List 36 | 37 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/git/GitActions.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.git 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.boot.context.event.ApplicationReadyEvent 5 | import org.springframework.context.ApplicationListener 6 | import org.springframework.stereotype.Component 7 | import pe.proxy.proxybuilder2.database.ProxyRepository 8 | import pe.proxy.proxybuilder2.util.ProxyConfig 9 | import pe.proxy.proxybuilder2.util.Tasks 10 | import pe.proxy.proxybuilder2.util.Utils 11 | import pe.proxy.proxybuilder2.util.writer.CustomFileWriter 12 | import java.io.File 13 | import java.text.SimpleDateFormat 14 | import java.util.* 15 | import java.util.concurrent.Executors 16 | import java.util.concurrent.ScheduledExecutorService 17 | import java.util.concurrent.TimeUnit 18 | 19 | /** 20 | * GitActions 21 | * 22 | * @author Kai 23 | * @version 1.0, 19/05/2022 24 | */ 25 | @Component 26 | class GitActions(private val repository: ProxyRepository, 27 | private val config: ProxyConfig) : ApplicationListener { 28 | 29 | private val logger = LoggerFactory.getLogger(GitActions::class.java) 30 | 31 | private val shell = Shell(config) 32 | 33 | private val executor : ScheduledExecutorService = Executors.newScheduledThreadPool(2) 34 | 35 | override fun onApplicationEvent(event : ApplicationReadyEvent) { 36 | executor.scheduleAtFixedRate({ initialize() },50,90, TimeUnit.MINUTES) 37 | } 38 | 39 | fun initialize() { 40 | Tasks.thread.gitActions?.let { Tasks.thread.pauseAllExcept(it) } 41 | //Heavy task, requesting other threads to pause until this task has been completed 42 | val largeArchiveFile = File(config.outputPath + "/archive/json/proxies-archive.json") 43 | while(largeArchiveFile.lastModified() <= Utils.timestampMinus(30).time) { 44 | val task = Runnable { 45 | CustomFileWriter(repository, config).initialize() 46 | } 47 | executor.submit(task).get() 48 | Thread.sleep(10000L) 49 | } 50 | Tasks.thread.resumeAll() 51 | 52 | GitStage.values() 53 | .filter { it.command.isNotEmpty() } 54 | .forEach { nextAction(it) } 55 | 56 | logger.info(GitStage.COMPLETE.name) 57 | 58 | } 59 | 60 | private fun nextAction(stage : GitStage) { 61 | val command = when (stage) { 62 | GitStage.COMMITTING -> { 63 | stage.command 64 | .plus("Updated-${ SimpleDateFormat("dd/MM/yyyy-HH:mm:ss").format(Date()) }") 65 | } 66 | else -> { stage.command } 67 | } 68 | shell.parseCommand(command) 69 | 70 | logger.info("${stage.name} -> $command") 71 | } 72 | 73 | enum class GitStage(val command : Array) { 74 | 75 | NOT_RUNNING(emptyArray()), 76 | RUNNING(arrayOf("git", "add", ".")), 77 | COMMITTING(arrayOf("git", "commit", "-m")), 78 | PUSHING(arrayOf("git", "push", "origin", "main")), 79 | COMPLETE(emptyArray()) 80 | 81 | } 82 | 83 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/git/Shell.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.git 2 | 3 | import org.slf4j.LoggerFactory 4 | import pe.proxy.proxybuilder2.util.ProxyConfig 5 | import pe.proxy.proxybuilder2.util.Utils 6 | 7 | /** 8 | * Shell 9 | * 10 | * @author Kai 11 | * @version 1.0, 19/05/2022 12 | */ 13 | class Shell(val config : ProxyConfig) { 14 | 15 | private val logger = LoggerFactory.getLogger(Shell::class.java) 16 | 17 | fun parseCommand(gitArguments : Array) { 18 | 19 | val directory = arrayOf("cd \"${config.outputPath}\" && ") 20 | 21 | val command : Array = if(Utils.IS_WINDOWS) { 22 | val console = (arrayOf("cmd", "/c"/*, "start", "cmd", "/k"*/)) 23 | console.plus(directory).plus(gitArguments) 24 | } else { 25 | val console = (arrayOf("/bin/sh", "-c")) 26 | console.plus(directory.joinToString { it } + gitArguments.joinToString(" ") { it }) 27 | } 28 | 29 | execute(command) 30 | } 31 | 32 | private fun execute(command : Array) { 33 | val reader = Runtime.getRuntime().exec(command).inputStream.reader() 34 | logger.info(reader.readText()) 35 | Thread.sleep(60000) //Wait 60 seconds in-between commands 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/monitor/CachingMonitor.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.monitor 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.cache.annotation.CacheEvict 5 | import org.springframework.scheduling.annotation.EnableScheduling 6 | import org.springframework.scheduling.annotation.Scheduled 7 | import org.springframework.stereotype.Component 8 | 9 | /** 10 | * CachingMonitor 11 | * 12 | * @author Kai 13 | * @version 1.0, 09/06/2022 14 | */ 15 | @Component 16 | @EnableScheduling 17 | class CachingMonitor { 18 | 19 | private val logger = LoggerFactory.getLogger(CachingMonitor::class.java) 20 | 21 | @CacheEvict(allEntries = true, value = ["ProxyRepository_lastSuccess", "ProxyController_proxies"]) 22 | @Scheduled(fixedDelay = (1000 * 60) * 15, initialDelay = 1000) 23 | fun flush() { //Runs every 15 minutes 24 | logger.info("Cache has been flushed") 25 | } 26 | 27 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/monitor/ChecksMonitor.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.monitor 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.boot.context.event.ApplicationReadyEvent 5 | import org.springframework.context.ApplicationListener 6 | import org.springframework.stereotype.Component 7 | import pe.proxy.proxybuilder2.util.ProxyConfig 8 | import java.util.concurrent.Executors 9 | import java.util.concurrent.ScheduledExecutorService 10 | import java.util.concurrent.TimeUnit 11 | 12 | /** 13 | * ChecksMonitor 14 | * 15 | * @author Kai 16 | * @version 1.0, 29/05/2022 17 | */ 18 | @Component 19 | class ChecksMonitor(private val config : ProxyConfig) : ApplicationListener { 20 | 21 | private val logger = LoggerFactory.getLogger(ChecksMonitor::class.java) 22 | 23 | private val executor : ScheduledExecutorService = Executors.newScheduledThreadPool(1) 24 | 25 | companion object { var proxyListSize = 0; var currentIndex = 0 } 26 | 27 | override fun onApplicationEvent(event: ApplicationReadyEvent) { 28 | if(config.enabledThreads.checksPerSecond) 29 | executor.scheduleAtFixedRate( { initialize() }, 0, 1, TimeUnit.SECONDS ) 30 | } 31 | 32 | private var checksPerSecond = 0 33 | private var currentTime = 0 34 | private var remainingTime = "00:00:00" 35 | 36 | private fun initialize() { 37 | val remaining = proxyListSize - currentIndex 38 | 39 | currentTime++ 40 | 41 | if (currentIndex == 0) 42 | return 43 | 44 | checksPerSecond = currentIndex / currentTime 45 | 46 | val remainingAmount: Int = remaining / checksPerSecond 47 | 48 | remainingTime = String.format( 49 | "%d:%02d:%02d", remainingAmount / 3600, remainingAmount % 3600 / 60, remainingAmount % 60) 50 | 51 | logger.info("Time Remaining[$remainingTime] CPS[$checksPerSecond], ListSize[$proxyListSize]," + 52 | " Tested[$currentIndex], Remaining[$remaining]\r") 53 | } 54 | 55 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/monitor/EndpointMonitor.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.monitor 2 | 3 | import com.twilio.Twilio 4 | import com.twilio.rest.api.v2010.account.Message 5 | import com.twilio.type.PhoneNumber 6 | import io.netty.bootstrap.Bootstrap 7 | import io.netty.channel.ChannelInitializer 8 | import io.netty.channel.ChannelOption 9 | import io.netty.channel.nio.NioEventLoopGroup 10 | import io.netty.channel.socket.SocketChannel 11 | import io.netty.channel.socket.nio.NioSocketChannel 12 | import io.netty.handler.logging.LogLevel 13 | import io.netty.handler.logging.LoggingHandler 14 | import io.netty.handler.timeout.IdleStateHandler 15 | import org.slf4j.LoggerFactory 16 | import org.springframework.boot.context.event.ApplicationReadyEvent 17 | import org.springframework.context.ApplicationListener 18 | import org.springframework.stereotype.Component 19 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelData 20 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelEncoderDecoder 21 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelHandler 22 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelResponseData 23 | import pe.proxy.proxybuilder2.util.ProxyConfig 24 | import pe.proxy.proxybuilder2.util.Tasks 25 | import java.net.InetSocketAddress 26 | import java.util.concurrent.Executors 27 | import java.util.concurrent.ScheduledExecutorService 28 | import java.util.concurrent.TimeUnit 29 | import java.util.concurrent.atomic.AtomicBoolean 30 | 31 | /** 32 | * EndpointMonitor 33 | * 34 | * @author Kai 35 | * @version 1.0, 25/05/2022 36 | */ 37 | @Component 38 | class EndpointMonitor(val config : ProxyConfig) : ApplicationListener { 39 | 40 | private val logger = LoggerFactory.getLogger(EndpointMonitor::class.java) 41 | 42 | private val executor : ScheduledExecutorService = Executors.newScheduledThreadPool(1) 43 | 44 | private val workerGroup = NioEventLoopGroup() 45 | 46 | companion object { 47 | val connected = AtomicBoolean(false) 48 | } 49 | 50 | private val pause = Tasks.thread.endpointMonitor?.pause!! 51 | 52 | override fun onApplicationEvent(event: ApplicationReadyEvent) { 53 | if(config.enabledThreads.endpointMonitor) 54 | executor.scheduleAtFixedRate( { initialize() }, 0, 1, TimeUnit.HOURS ) 55 | } 56 | 57 | fun initialize() { 58 | if(pause.get()) 59 | return logger.info("Thread paused") 60 | 61 | for (endpoint in config.endpointServers) { 62 | if (endpoint.name.startsWith("!")) 63 | continue 64 | val connect = connect(endpoint) //ProxyConfig.EndpointServer("aws_na", "123.123.123.123", 43594) 65 | if(!connect) { 66 | sendText(endpoint) 67 | } else { 68 | connected.set(false) 69 | logger.info("Connected successfully to endpoint -> $endpoint") 70 | } 71 | Thread.sleep(30000L) //Wait 30 seconds before looping to next endpoint 72 | } 73 | logger.info("Finished this task") 74 | } 75 | 76 | fun sendText(endpoint : ProxyConfig.EndpointServer) { 77 | val t = config.twilio 78 | 79 | Twilio.init(t.sid, t.token) 80 | val message : Message = Message.creator( 81 | PhoneNumber(t.phoneNumberTo), 82 | PhoneNumber(t.phoneNumberFrom), 83 | "$endpoint is currently down.") 84 | .create() 85 | 86 | logger.info("Unable to connect to endpoint -> $endpoint | ${message.status}") 87 | } 88 | 89 | fun connect(endpoint : ProxyConfig.EndpointServer) : Boolean { 90 | try { 91 | Bootstrap().group(workerGroup) 92 | .channel(NioSocketChannel::class.java) 93 | .option(ChannelOption.AUTO_READ, true) 94 | .option(ChannelOption.TCP_NODELAY, true) 95 | .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.timeout) 96 | .handler(object : ChannelInitializer() { 97 | override fun initChannel(channel: SocketChannel) { 98 | val direct = ProxyChannelData( 99 | "0.0.0.0", 80, "", "", "", true, 100 | endpoint, ProxyChannelResponseData() 101 | ) 102 | val encoderDecoder = ProxyChannelEncoderDecoder(direct) 103 | val handler = ProxyChannelHandler(encoderDecoder) 104 | channel.pipeline() 105 | .addLast(LoggingHandler(LogLevel.DEBUG)) 106 | .addLast(IdleStateHandler(0, 0, 10)) 107 | .addLast(handler) 108 | } 109 | }) 110 | .connect(InetSocketAddress(endpoint.ip, endpoint.port)) 111 | .channel().closeFuture().sync() 112 | 113 | //Sleep for 5s, in-case waiting for connected boolean to bet set 114 | Thread.sleep(5000L) 115 | return connected.get() 116 | } catch (e: Exception) { 117 | logger.error(e.localizedMessage) 118 | } 119 | return false 120 | } 121 | 122 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/monitor/SQLProxyMonitor.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.monitor 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.boot.context.event.ApplicationReadyEvent 5 | import org.springframework.context.ApplicationListener 6 | import org.springframework.stereotype.Component 7 | import pe.proxy.proxybuilder2.database.ProxyInteraction 8 | import pe.proxy.proxybuilder2.database.ProxyRepository 9 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyChannelData 10 | import pe.proxy.proxybuilder2.net.proxy.tester.ProxyConnect 11 | import pe.proxy.proxybuilder2.util.ProxyConfig 12 | import pe.proxy.proxybuilder2.util.Tasks 13 | import java.util.concurrent.ConcurrentLinkedQueue 14 | import java.util.concurrent.Executors 15 | import java.util.concurrent.ScheduledExecutorService 16 | import java.util.concurrent.TimeUnit 17 | 18 | /** 19 | * SQLProxyMonitor 20 | * 21 | * Updates MariaDB every 15 seconds with proxies supplied from ProxyConnect() 22 | * 23 | * @author Kai 24 | * @version 1.0, 23/05/2022 25 | */ 26 | @Component 27 | class SQLProxyMonitor(val repository : ProxyRepository, val config : ProxyConfig) : ApplicationListener { 28 | 29 | private val logger = LoggerFactory.getLogger(SQLProxyMonitor::class.java) 30 | 31 | private val executor : ScheduledExecutorService = Executors.newScheduledThreadPool(2) 32 | 33 | private val running = Tasks.thread.sqlProxyMonitor?.running!! 34 | private val pause = Tasks.thread.proxyConnect?.pause!! 35 | 36 | override fun onApplicationEvent(event: ApplicationReadyEvent) { 37 | if(config.enabledThreads.sqlProxyMonitor) 38 | executor.scheduleAtFixedRate( { initialize() }, 1, 1, TimeUnit.SECONDS ) 39 | } 40 | 41 | fun initialize() = try { 42 | if (!running.get()) 43 | filter() 44 | else 45 | logger.info("Task is not ready") 46 | } catch (e : Exception) { 47 | logger.error(e.localizedMessage) 48 | } 49 | 50 | //Filters the proxies into a list then calls the "write(proxy)" function 51 | fun filter() { 52 | if(pause.get()) 53 | return logger.info("Thread paused") 54 | 55 | running.set(true) 56 | val proxies = ProxyConnect.testedProxies 57 | if(proxies.isNotEmpty()) 58 | writeAll(proxies) 59 | 60 | running.set(false) 61 | } 62 | 63 | //Writes proxy to database 64 | fun write(proxy : ProxyChannelData) { 65 | val interaction = ProxyInteraction(repository) 66 | val entity = interaction.getProxyEntity(proxy) 67 | if(entity != null) 68 | interaction.updateEntity(entity, proxy) 69 | } 70 | 71 | fun writeAll(proxies : ConcurrentLinkedQueue) { 72 | val interaction = ProxyInteraction(repository) 73 | val entities = interaction.getProxyEntities(proxies) 74 | if(entities.isNotEmpty()) 75 | interaction.updateEntities(entities) 76 | } 77 | 78 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/data/ProxyData.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.data 2 | 3 | import kotlinx.serialization.Serializable 4 | import pe.proxy.proxybuilder2.util.Utils 5 | 6 | /** 7 | * ProxyData 8 | * 9 | * @author Kai 10 | * @version 1.0, 19/05/2022 11 | */ 12 | @Serializable 13 | data class SupplierProxyListData(val http : MutableList, val https : MutableList, 14 | val socks4 : MutableList, val socks5 : MutableList) { 15 | fun size() : Int = http.size + https.size + socks4.size + socks5.size 16 | 17 | //TODO - Improve performance/readability 18 | fun removeAtRandom(amount : Int) { 19 | val lists = mutableListOf>() 20 | when { http.size > 0 -> lists.add(http) } 21 | when { https.size > 0 -> lists.add(https) } 22 | when { socks4.size > 0 -> lists.add(socks4) } 23 | when { socks5.size > 0 -> lists.add(socks5) } 24 | for(i in 0 until amount) { 25 | val theList = lists[Utils.GLOBAL_RANDOM.nextInt(lists.size)] 26 | if(theList.isNotEmpty()) 27 | theList.removeAt(0) 28 | else 29 | removeAtRandom(1) //Recurse 30 | } 31 | } 32 | } 33 | 34 | @Serializable 35 | data class SimpleProxyDataList(val proxies : MutableMap) { 36 | fun size(protocol : String) : Int = proxies.filter { it.value.protocol == protocol }.size 37 | 38 | } 39 | 40 | @Serializable 41 | data class SimpleProxyDataType(val protocol : String, val ip : String, val port : Int) 42 | 43 | @Serializable 44 | data class PerformanceConnectData(val aws_NA : EndpointServerData?=null, val ora_UK : EndpointServerData?=null, 45 | val ora_JP : EndpointServerData?=null, val ms_HK : EndpointServerData?=null) { 46 | fun default() : PerformanceConnectData = PerformanceConnectData( 47 | EndpointServerData().default(), EndpointServerData().default(), 48 | EndpointServerData().default(), EndpointServerData().default() 49 | ) 50 | 51 | } 52 | 53 | @Serializable 54 | data class EndpointServerData(var ping : Long?=0L, val connections : ConnectionAttempts?=null, 55 | var uptime : String?=null) { 56 | fun default() : EndpointServerData = EndpointServerData(0, ConnectionAttempts(0, 0), "0%") 57 | } 58 | 59 | @Serializable 60 | data class ConnectionAttempts(var success : Int, var fail : Int) 61 | 62 | @Serializable 63 | data class ProtocolData(var protocol : MutableList) 64 | 65 | @Serializable 66 | data class ProtocolDataType(var type : String, var port : Int, var tls : Boolean?=false, 67 | var autoRead : MutableList?=null) 68 | 69 | @Serializable 70 | data class ProxyCredentials(var username : String, var password : String) { 71 | fun empty() : Boolean = username.isEmpty() && password.isEmpty() 72 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/proxycheckio/QueryApi.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.proxycheckio 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.boot.context.event.ApplicationReadyEvent 5 | import org.springframework.context.ApplicationListener 6 | import org.springframework.stereotype.Component 7 | import pe.proxy.proxybuilder2.database.ProxyRepository 8 | import pe.proxy.proxybuilder2.util.KotlinSerializer 9 | import pe.proxy.proxybuilder2.util.ProxyConfig 10 | import pe.proxy.proxybuilder2.util.Tasks 11 | import pe.proxy.proxybuilder2.util.Utils 12 | import java.net.URI 13 | import java.net.http.HttpClient 14 | import java.net.http.HttpRequest 15 | import java.net.http.HttpResponse 16 | import java.time.Duration 17 | import java.util.concurrent.Executors 18 | import java.util.concurrent.ScheduledExecutorService 19 | import java.util.concurrent.TimeUnit 20 | 21 | 22 | /** 23 | * QueryApi 24 | * 25 | * @author Kai 26 | * @version 1.0, 21/05/2022 27 | */ 28 | @Component 29 | class QueryApi(private val proxyRepository : ProxyRepository, 30 | private val config : ProxyConfig) : ApplicationListener { 31 | 32 | private val logger = LoggerFactory.getLogger(QueryApi::class.java) 33 | 34 | private val client : HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() 35 | 36 | private val builder : HttpRequest.Builder = HttpRequest.newBuilder() 37 | 38 | private val executor : ScheduledExecutorService = Executors.newScheduledThreadPool(1) 39 | 40 | private val pause = Tasks.thread.queryApi?.pause!! 41 | 42 | override fun onApplicationEvent(event : ApplicationReadyEvent) { 43 | //Runs query() every minute 44 | if(config.enabledThreads.queryApi) 45 | executor.scheduleAtFixedRate({ initialize() }, 15, 45, TimeUnit.SECONDS) 46 | } 47 | 48 | fun initialize() { 49 | try { 50 | if(pause.get()) 51 | return logger.info("Thread paused") 52 | 53 | logger.info("Querying ProxyCheck API") 54 | 55 | val entitiesFromRepository = proxyRepository.findByLocationIsNullAndLastSuccessIsNotNull() 56 | 57 | if (entitiesFromRepository.isEmpty()) 58 | return logger.info("Repository is empty") 59 | 60 | val maxSize = (if(entitiesFromRepository.size > 100) 100 else entitiesFromRepository.size) 61 | 62 | val entities = entitiesFromRepository.subList(0, maxSize) //Max 100 so that we don't overload the API 63 | 64 | for (entity in entities) { 65 | val entityIp = entity.ip ?: continue 66 | val request = builder.uri(apiURI(entityIp)) //Impossible for entity.ip to be null 67 | .GET() 68 | .timeout(Duration.ofSeconds(5)) 69 | .build() 70 | 71 | val json = client.sendAsync(request, HttpResponse.BodyHandlers.ofString()).get().body() 72 | val clazzes = Utils.serialize(json) 73 | 74 | clazzes.forEach { clazz -> 75 | when (clazz) { 76 | is LocationData -> { 77 | entity.location = KotlinSerializer.encode(clazz) 78 | entity.provider = clazz.provider //Default Provider 79 | } 80 | is RiskData -> entity.detection = KotlinSerializer.encode(clazz) 81 | is OperatorData -> { 82 | val policiesClazz = clazzes[clazzes.size - 1] 83 | if (policiesClazz is PoliciesData && !policiesClazz.isEmpty()) 84 | clazz.policies = policiesClazz 85 | if (!clazz.isEmpty()) 86 | entity.provider = 87 | KotlinSerializer.encode(clazz) //Overwrite Provider if more info available 88 | } 89 | } 90 | } 91 | } 92 | 93 | proxyRepository.saveAll(entities) 94 | logger.info("Query complete - ${entities.size}") 95 | 96 | } catch (e: Exception) { 97 | logger.error(e.localizedMessage) 98 | } 99 | } 100 | 101 | fun apiURI(proxyIp : String) : URI { 102 | return URI.create("http://proxycheck.io/v2/$proxyIp?" + 103 | "key=${config.proxyCheckIo.apiKey}&vpn=1&asn=1&risk=2&seen=1&tag=msg") 104 | //http://proxycheck.io/v2/80.90.80.54?key=7y658u-044228-737v80-64lq59" (Example IP/Key) 105 | } 106 | 107 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/proxycheckio/QueryData.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.proxycheckio 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore 4 | import kotlinx.serialization.Serializable 5 | 6 | /** 7 | * QueryData 8 | * 9 | * @author Kai 10 | * @version 1.0, 21/05/2022 11 | */ 12 | @Serializable 13 | data class LocationData(var continent : String?=null, var country : String?=null, 14 | var isocode : String?=null, var region : String?=null, var regioncode: String?=null, 15 | var city : String?=null, var latitude : Float?=null, var longitude : Float?=null, 16 | var provider : String?=null, var organisation : String?=null, var asn : String?=null) 17 | 18 | @Serializable 19 | data class OperatorData(var name: String?=null, var url: String?=null, var anonymity: String?=null, 20 | var popularity: String?=null, var policies: PoliciesData?=null) { 21 | @JsonIgnore 22 | fun isEmpty() : Boolean = name == null && url == null && anonymity == null 23 | && popularity == null && policies == null 24 | } 25 | 26 | @Serializable 27 | data class PoliciesData(var ad_filtering : Boolean?=null, var free_access : Boolean?=null, 28 | var paid_access : Boolean?=null, var port_forwarding : Boolean?=null, 29 | var logging : Boolean?=null, var anonymous_payments : Boolean?=null, 30 | var crypto_payments : Boolean?=null, var traceable_ownership : Boolean?=null) { 31 | @JsonIgnore 32 | fun isEmpty() : Boolean = ad_filtering == null && free_access == null && paid_access == null 33 | && port_forwarding == null && logging == null && anonymous_payments == null 34 | && crypto_payments == null && traceable_ownership == null 35 | } 36 | 37 | @Serializable 38 | data class RiskData(var proxy : Boolean?=null, var type : String?=null, var risk : Int?=null) -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/supplier/CustomProxySupplier.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.supplier 2 | 3 | import kotlinx.serialization.decodeFromString 4 | import kotlinx.serialization.json.Json 5 | import org.slf4j.LoggerFactory 6 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataList 7 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataType 8 | import pe.proxy.proxybuilder2.net.proxy.data.SupplierProxyListData 9 | import pe.proxy.proxybuilder2.util.ProxyConfig 10 | import java.net.URI 11 | import java.net.http.HttpClient 12 | import java.net.http.HttpRequest 13 | import java.net.http.HttpResponse 14 | import java.time.Duration 15 | 16 | /** 17 | * CustomProxySupplier 18 | * 19 | * @author Kai 20 | * @version 1.0, 15/05/2022 21 | */ 22 | class CustomProxySupplier(override val data : SimpleProxyDataList, appConfig : ProxyConfig) : IProxySupplier { 23 | 24 | private val logger = LoggerFactory.getLogger(CustomProxySupplier::class.java) 25 | 26 | private val endpointURL = appConfig.supplier.customUrl 27 | 28 | private val client : HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() 29 | private val builder : HttpRequest.Builder = HttpRequest.newBuilder() 30 | 31 | private var unparsed : String?= null 32 | 33 | override fun request() : CustomProxySupplier { 34 | val httpRequest = builder.uri(URI.create(endpointURL)).GET().timeout(Duration.ofSeconds(15)).build() 35 | this.unparsed = client.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString()).get().body() 36 | return this 37 | } 38 | 39 | //TODO Modify parsing for CustomProxySupplier 40 | override fun parse() { 41 | val unparsed = this.unparsed ?: return logger.error("Unable to read unparsed body data from supplier") 42 | 43 | val data = Json { this.encodeDefaults = true; this.ignoreUnknownKeys = true } 44 | val parsed = data.decodeFromString(unparsed) 45 | 46 | val hashMap = HashMap>() 47 | 48 | hashMap["http"] = parsed.http 49 | hashMap["https"] = parsed.https 50 | hashMap["socks4"] = parsed.socks4 51 | hashMap["socks5"] = parsed.socks5 52 | 53 | for ((key, proxyList) in hashMap) { 54 | for(proxy in proxyList) { 55 | if(!proxy.contains(":") || proxy.length > 23) 56 | continue 57 | val ip = proxy.split(":")[0] 58 | val port = proxy.split(":")[1] 59 | this.data.proxies[ip] = SimpleProxyDataType(key, ip, port.toInt()) 60 | } 61 | } 62 | 63 | logger.info( 64 | "Parsing complete -> " + 65 | "[HTTP:${this.data.size("http")} | HTTPS:${this.data.size("https")} | " + 66 | "SOCKS4:${this.data.size("socks4")} | SOCKS5:${this.data.size("socks5")}]" 67 | ) 68 | } 69 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/supplier/DatabaseProxySupplier.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.supplier 2 | 3 | import org.slf4j.LoggerFactory 4 | import pe.proxy.proxybuilder2.database.EntityForPublicView 5 | import pe.proxy.proxybuilder2.database.ProxyEntity 6 | import pe.proxy.proxybuilder2.database.ProxyRepository 7 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataList 8 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataType 9 | import java.sql.Timestamp 10 | import java.time.LocalDateTime 11 | 12 | /** 13 | * DatabaseProxySupplier 14 | * 15 | * @author Kai 16 | * @version 1.0, 31/05/2022 17 | */ 18 | class DatabaseProxySupplier( 19 | override val data: SimpleProxyDataList, 20 | private val repository: ProxyRepository, 21 | ) : IProxySupplier { 22 | 23 | private val logger = LoggerFactory.getLogger(DatabaseProxySupplier::class.java) 24 | 25 | private var proxyEntities : List?=null 26 | 27 | override fun request() : DatabaseProxySupplier { 28 | //Only test proxies that were last successful 6 months or less 29 | val timestamp = Timestamp.valueOf(LocalDateTime.now().minusMonths(6)) 30 | proxyEntities = repository.findByLastSuccessAfter(timestamp) 31 | logger.info("One example proxy is: ${proxyEntities?.get(0)}") 32 | return this 33 | } 34 | 35 | @Throws 36 | override fun parse() { 37 | val proxies = mutableListOf() 38 | 39 | proxyEntities?.mapTo(proxies) { 40 | EntityForPublicView().minimal(it) 41 | } 42 | 43 | val protocols = listOf("http", "https", "socks4", "socks5") 44 | for(protocolName in protocols) { 45 | proxies.flatMap { prox -> 46 | prox.protocols 47 | ?.map { repo -> prox to repo } 48 | ?.filter { it.second.type == protocolName }!! 49 | }.distinctBy { listOf(it.first.ip, it.first.port) }.forEach { 50 | data.proxies[it.first.ip!!] = SimpleProxyDataType(protocolName, it.first.ip!!, it.first.port!!) 51 | } 52 | } 53 | 54 | logger.info( 55 | "Parsing complete -> " + 56 | "[HTTP:${this.data.size("http")} | HTTPS:${this.data.size("https")} | " + 57 | "SOCKS4:${this.data.size("socks4")} | SOCKS5:${this.data.size("socks5")}]" 58 | ) 59 | } 60 | 61 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/supplier/IProxySupplier.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.supplier 2 | 3 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataList 4 | 5 | /** 6 | * IProxySupplier 7 | * 8 | * @author Kai 9 | * @version 1.0, 15/05/2022 10 | */ 11 | interface IProxySupplier { 12 | 13 | val data : SimpleProxyDataList 14 | fun request() : IProxySupplier 15 | fun parse() 16 | 17 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/supplier/LocalProxySupplier.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.supplier 2 | 3 | import kotlinx.serialization.decodeFromString 4 | import kotlinx.serialization.json.Json 5 | import org.slf4j.LoggerFactory 6 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataList 7 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataType 8 | import pe.proxy.proxybuilder2.net.proxy.data.SupplierProxyListData 9 | import pe.proxy.proxybuilder2.util.ProxyConfig 10 | import pe.proxy.proxybuilder2.util.Utils 11 | import java.io.File 12 | import java.nio.file.Path 13 | import java.nio.file.Paths 14 | 15 | /** 16 | * LocalProxySupplier 17 | * 18 | * @author Kai 19 | * @version 1.0, 15/05/2022 20 | */ 21 | class LocalProxySupplier(override val data : SimpleProxyDataList, appConfig : ProxyConfig) : IProxySupplier { 22 | 23 | private val logger = LoggerFactory.getLogger(LocalProxySupplier::class.java) 24 | 25 | private val endpointURL = appConfig.supplier.customUrl 26 | 27 | override fun request() : LocalProxySupplier { 28 | return this 29 | } 30 | 31 | //TODO Modify parsing for CustomProxySupplier 32 | override fun parse() { 33 | 34 | val data = Json { this.encodeDefaults = true; this.ignoreUnknownKeys = true } 35 | 36 | val parsed = SupplierProxyListData(mutableListOf(), mutableListOf(), mutableListOf(), mutableListOf()) 37 | 38 | val path: Path = if(Utils.IS_WINDOWS) { 39 | Paths.get("proxies/") 40 | } else { 41 | Paths.get("/home/proxybuilder/IntelliJProjects/proxy-builder-2/proxies/") 42 | } 43 | 44 | File(path.toUri()).listFiles()?.forEach { 45 | if(it.extension.contains("json")) { 46 | val prox = it?.readText()?.let { it1 -> data.decodeFromString(it1) } 47 | if(prox != null) { 48 | parsed.http.addAll(prox.http) 49 | parsed.https.addAll(prox.https) 50 | parsed.socks4.addAll(prox.socks4) 51 | parsed.socks5.addAll(prox.socks5) 52 | } 53 | } 54 | } 55 | 56 | val hashMap = HashMap>() 57 | 58 | hashMap["http"] = parsed.http 59 | hashMap["https"] = parsed.https 60 | hashMap["socks4"] = parsed.socks4 61 | hashMap["socks5"] = parsed.socks5 62 | 63 | 64 | 65 | for ((key, proxyList) in hashMap) { 66 | for(proxy in proxyList) { 67 | if(!proxy.contains(":") || proxy.length > 23) 68 | continue 69 | val ip = proxy.split(":")[0] 70 | val port = proxy.split(":")[1] 71 | this.data.proxies[ip] = SimpleProxyDataType(key, ip, port.toInt()) 72 | } 73 | } 74 | 75 | logger.info( 76 | "Parsing complete -> " + 77 | "[HTTP:${this.data.size("http")} | HTTPS:${this.data.size("https")} | " + 78 | "SOCKS4:${this.data.size("socks4")} | SOCKS5:${this.data.size("socks5")}]" 79 | ) 80 | } 81 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/supplier/MainProxySupplier.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.supplier 2 | 3 | import kotlinx.serialization.decodeFromString 4 | import kotlinx.serialization.json.Json 5 | import org.slf4j.LoggerFactory 6 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataList 7 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataType 8 | import pe.proxy.proxybuilder2.net.proxy.data.SupplierProxyListData 9 | import pe.proxy.proxybuilder2.util.ProxyConfig 10 | import java.net.URI 11 | import java.net.http.HttpClient 12 | import java.net.http.HttpRequest 13 | import java.net.http.HttpResponse 14 | import java.time.Duration 15 | 16 | /** 17 | * MainProxySupplier 18 | * 19 | * @author Kai 20 | * @version 1.0, 15/05/2022 21 | */ 22 | class MainProxySupplier(override val data : SimpleProxyDataList, appConfig: ProxyConfig) : IProxySupplier { 23 | 24 | private val logger = LoggerFactory.getLogger(MainProxySupplier::class.java) 25 | 26 | private val endpointURL = appConfig.supplier.mainUrl 27 | 28 | private val client : HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() 29 | private val builder : HttpRequest.Builder = HttpRequest.newBuilder() 30 | 31 | private var unparsed : String ?= null 32 | 33 | override fun request() : MainProxySupplier { 34 | val httpRequest = builder.uri(URI.create(endpointURL)).GET().timeout(Duration.ofSeconds(15)).build() 35 | this.unparsed = client.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString()).get().body() 36 | return this 37 | } 38 | 39 | override fun parse() { 40 | val unparsed = this.unparsed ?: return 41 | val data = Json { this.encodeDefaults = true; this.ignoreUnknownKeys = true } 42 | val parsed = data.decodeFromString(unparsed) 43 | 44 | val hashMap = HashMap>() 45 | 46 | hashMap["http"] = parsed.http 47 | hashMap["https"] = parsed.https 48 | hashMap["socks4"] = parsed.socks4 49 | hashMap["socks5"] = parsed.socks5 50 | 51 | for ((key, proxyList) in hashMap) { 52 | for(proxy in proxyList) { 53 | if(!proxy.contains(":") || proxy.length > 23) 54 | continue 55 | val ip = proxy.split(":")[0] 56 | val port = proxy.split(":")[1] 57 | this.data.proxies[ip] = SimpleProxyDataType(key, ip, port.toInt()) 58 | } 59 | } 60 | 61 | logger.info( 62 | "Parsing complete -> " + 63 | "[HTTP:${this.data.size("http")} | HTTPS:${this.data.size("https")} | " + 64 | "SOCKS4:${this.data.size("socks4")} | SOCKS5:${this.data.size("socks5")}]" 65 | ) 66 | } 67 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/tester/ProxyChannelData.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.tester 2 | 3 | import pe.proxy.proxybuilder2.util.ProxyConfig 4 | import java.sql.Timestamp 5 | 6 | /** 7 | * ProxyChannelData 8 | * 9 | * @author Kai 10 | * @version 1.0, 19/05/2022 11 | */ 12 | data class ProxyChannelData(val ip : String, val port : Int, val type : String, 13 | val username : String, val password : String, val autoRead : Boolean, 14 | val endpointServer : ProxyConfig.EndpointServer?=null, val response : ProxyChannelResponseData) { 15 | fun remoteAddress() : String = "$ip:$port" 16 | 17 | } 18 | 19 | data class ProxyChannelResponseData(var connected : Boolean?=false, var tls : Boolean?=false, 20 | var readable : Boolean?=false, var remoteIp : String?=null, 21 | var startTime : Timestamp?=null, var endTime : Timestamp?=null, 22 | var autoRead : MutableList?=null) -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/tester/ProxyChannelEncoderDecoder.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.tester 2 | 3 | import io.netty.buffer.ByteBuf 4 | import io.netty.buffer.Unpooled 5 | import io.netty.channel.ChannelHandlerContext 6 | import org.slf4j.LoggerFactory 7 | import pe.proxy.proxybuilder2.util.Utils 8 | 9 | /** 10 | * ProxyChannelEncoderDecoder 11 | * 12 | * @author Kai 13 | * @version 1.0, 19/05/2022 14 | */ 15 | class ProxyChannelEncoderDecoder(val proxy : ProxyChannelData) { 16 | 17 | private val logger = LoggerFactory.getLogger(ProxyChannelEncoderDecoder::class.java) 18 | 19 | fun encode(ctx : ChannelHandlerContext) { 20 | proxy.response.startTime = Utils.timestampNow() 21 | //Sending two bytes to the endpoint, 14 & 0 22 | val buffer = Unpooled.buffer(2) 23 | buffer.writeByte(14) 24 | buffer.writeByte(0) 25 | ctx.channel().writeAndFlush(buffer) 26 | } 27 | 28 | fun decode(ctx : ChannelHandlerContext, buffer : ByteBuf) { 29 | //These are the Integers that we are expecting to come from the endpoint 30 | val values = intArrayOf(0xFFFFFF, 0x000000, 0xFF00FF, 0x00FF00, 0xFFFF00, 0x00FFFF) 31 | val readable = buffer.isReadable 32 | 33 | //proxy.response.tls = values.all { buffer.isReadable && buffer.readInt() == it } 34 | 35 | //Ensure that the Integers match, if so then set readable to $true 36 | if(buffer.readableBytes() == values.size * 4) { 37 | for ((index, value) in values.withIndex()) { 38 | if(!buffer.isReadable) 39 | break 40 | 41 | val nextInt = buffer.readInt() 42 | if(value != nextInt) 43 | break 44 | 45 | if (values.size - 1 == index) 46 | proxy.response.tls = true 47 | } 48 | } 49 | 50 | val remoteIp = Utils.splitRemoteIp(ctx.channel().remoteAddress().toString()) 51 | 52 | //Check if data is readable & ips match, update database entry if proxy is working 53 | if(readable) { 54 | if (Utils.ipMatch(proxy.ip, remoteIp)) { 55 | val autoReadList = mutableListOf() 56 | autoReadList.add(proxy.autoRead) 57 | 58 | val response = proxy.response 59 | response.readable = true 60 | response.remoteIp = remoteIp 61 | response.connected = true 62 | response.autoRead = autoReadList 63 | response.endTime = Utils.timestampNow() 64 | } else { 65 | logger.warn("Proxy ip does not match ${proxy.ip} -> $remoteIp") 66 | } 67 | } else { 68 | logger.warn("Response data does not match -> ${proxy.remoteAddress()}") 69 | } 70 | 71 | buffer.release() 72 | } 73 | 74 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/tester/ProxyChannelHandler.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.tester 2 | 3 | import io.netty.buffer.ByteBuf 4 | import io.netty.buffer.Unpooled 5 | import io.netty.channel.ChannelFutureListener 6 | import io.netty.channel.ChannelHandlerContext 7 | import io.netty.channel.SimpleChannelInboundHandler 8 | import io.netty.handler.timeout.IdleState 9 | import io.netty.handler.timeout.IdleStateEvent 10 | import org.slf4j.LoggerFactory 11 | import pe.proxy.proxybuilder2.monitor.EndpointMonitor 12 | 13 | /** 14 | * ProxyChannelHandler 15 | * 16 | * @author Kai 17 | * @version 1.0, 19/05/2022 18 | */ 19 | class ProxyChannelHandler(private val encoderDecoder : ProxyChannelEncoderDecoder) 20 | : SimpleChannelInboundHandler() { 21 | 22 | private val logger = LoggerFactory.getLogger(ProxyChannelHandler::class.java) 23 | 24 | private var existingConnection = false 25 | 26 | override fun channelUnregistered(ctx : ChannelHandlerContext) { //Finished 27 | if(encoderDecoder.proxy.ip == "0.0.0.0") { 28 | EndpointMonitor.connected.set(encoderDecoder.proxy.response.tls == true) 29 | } else if(!existingConnection) { 30 | ProxyConnect.testedProxies.offer(encoderDecoder.proxy) 31 | } 32 | super.channelUnregistered(ctx) 33 | } 34 | 35 | override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { 36 | if(evt is IdleStateEvent) { 37 | val state = evt.state() 38 | if(state == IdleState.ALL_IDLE) { 39 | //logger.info("State is idle, closing connection") 40 | ctx.close() 41 | } 42 | } else 43 | super.userEventTriggered(ctx, evt) 44 | } 45 | 46 | override fun channelActive(ctx : ChannelHandlerContext) { 47 | encoderDecoder.encode(ctx) 48 | } 49 | 50 | override fun channelRead(ctx : ChannelHandlerContext, msg : Any) { 51 | if(msg is ByteBuf) 52 | encoderDecoder.decode(ctx, msg) 53 | flushAndClose(ctx) 54 | } 55 | 56 | override fun channelRead0(ctx: ChannelHandlerContext, msg: String) { 57 | super.channelRead(ctx, msg) 58 | } 59 | 60 | override fun channelInactive(ctx : ChannelHandlerContext) { 61 | flushAndClose(ctx) 62 | } 63 | 64 | @Deprecated("TODO -> This will be deprecated in a future Netty build") 65 | override fun exceptionCaught(ctx : ChannelHandlerContext, cause : Throwable) { 66 | val messages = listOf("Too many open connections") 67 | if(messages.contains(cause.localizedMessage)) 68 | existingConnection = true 69 | 70 | flushAndClose(ctx) 71 | //logger.warn(cause.localizedMessage) 72 | } 73 | 74 | private fun flushAndClose(ctx : ChannelHandlerContext) { 75 | val channel = ctx.channel() 76 | if (channel.isActive) 77 | channel.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE) 78 | else 79 | ctx.close() 80 | } 81 | 82 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/tester/ProxyChannelInitializer.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.tester 2 | 3 | import io.netty.channel.ChannelInitializer 4 | import io.netty.channel.socket.SocketChannel 5 | import io.netty.handler.logging.LogLevel 6 | import io.netty.handler.logging.LoggingHandler 7 | import io.netty.handler.proxy.HttpProxyHandler 8 | import io.netty.handler.proxy.ProxyHandler 9 | import io.netty.handler.proxy.Socks4ProxyHandler 10 | import io.netty.handler.proxy.Socks5ProxyHandler 11 | import io.netty.handler.timeout.IdleStateHandler 12 | import org.slf4j.LoggerFactory 13 | import java.net.InetSocketAddress 14 | 15 | /** 16 | * ProxyChannelInitializer 17 | * 18 | * @author Kai 19 | * @version 1.0, 19/05/2022 20 | */ 21 | class ProxyChannelInitializer(private val proxy : ProxyChannelData) : ChannelInitializer() { 22 | 23 | private val logger = LoggerFactory.getLogger(ProxyChannelInitializer::class.java) 24 | 25 | override fun initChannel(channel : SocketChannel) { 26 | val proxyHandler = proxyHandler() 27 | if (proxyHandler == null) { 28 | logger.error("Protocol not specified") 29 | channel.close() 30 | } else { 31 | proxyHandler.setConnectTimeoutMillis(10000L) 32 | val encoderDecoder = ProxyChannelEncoderDecoder(proxy) 33 | val handler = ProxyChannelHandler(encoderDecoder) 34 | 35 | channel.pipeline() 36 | .addLast(LoggingHandler(LogLevel.DEBUG)) 37 | .addLast(IdleStateHandler(0, 0, 10)) 38 | .addLast(proxyHandler) 39 | .addLast(handler) 40 | 41 | /* if(proxyHandler is HttpProxyHandler) { //TODO 42 | pipeline.addLast(HttpRequestEncoder()) 43 | .addLast(HttpObjectAggregator(8192)) 44 | .addLast(HttpResponseDecoder()) 45 | }*/ 46 | } 47 | } 48 | 49 | private fun proxyHandler() : ProxyHandler? { 50 | val proxyAddress = InetSocketAddress(proxy.ip, proxy.port) 51 | return when (proxy.type) { 52 | "socks4" -> { Socks4ProxyHandler(proxyAddress, proxy.username) } 53 | "socks5" -> { Socks5ProxyHandler(proxyAddress, proxy.username, proxy.password) } 54 | "http", "https" -> { HttpProxyHandler(proxyAddress, proxy.username, proxy.password) } 55 | else -> null 56 | } 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/net/proxy/tester/ProxyConnect.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.net.proxy.tester 2 | 3 | import io.netty.bootstrap.Bootstrap 4 | import io.netty.channel.ChannelOption 5 | import io.netty.channel.nio.NioEventLoopGroup 6 | import io.netty.channel.socket.nio.NioSocketChannel 7 | import io.netty.resolver.NoopAddressResolverGroup 8 | import org.slf4j.LoggerFactory 9 | import org.springframework.boot.context.event.ApplicationReadyEvent 10 | import org.springframework.context.ApplicationListener 11 | import org.springframework.stereotype.Component 12 | import pe.proxy.proxybuilder2.database.ProxyRepository 13 | import pe.proxy.proxybuilder2.monitor.ChecksMonitor 14 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataList 15 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataType 16 | import pe.proxy.proxybuilder2.net.proxy.supplier.DatabaseProxySupplier 17 | import pe.proxy.proxybuilder2.net.proxy.supplier.LocalProxySupplier 18 | import pe.proxy.proxybuilder2.net.proxy.supplier.MainProxySupplier 19 | import pe.proxy.proxybuilder2.util.ProxyConfig 20 | import pe.proxy.proxybuilder2.util.Tasks 21 | import pe.proxy.proxybuilder2.util.Utils 22 | import java.net.InetSocketAddress 23 | import java.sql.Timestamp 24 | import java.time.LocalDateTime 25 | import java.util.concurrent.ConcurrentLinkedQueue 26 | import java.util.concurrent.ExecutorService 27 | import java.util.concurrent.Executors 28 | 29 | /** 30 | * ProxyConnect 31 | * 32 | * @author Kai 33 | * @version 1.0, 19/05/2022 34 | */ 35 | @Component 36 | class ProxyConnect(val repository : ProxyRepository, final val config : ProxyConfig) : ApplicationListener { 37 | 38 | private val logger = LoggerFactory.getLogger(ProxyConnect::class.java) 39 | 40 | private val workerGroup = NioEventLoopGroup(config.threads) 41 | 42 | private val executor : ExecutorService = Executors.newFixedThreadPool( 43 | Runtime.getRuntime().availableProcessors() + 1 44 | ) 45 | 46 | companion object { //TODO - CHANGE THIS 47 | val testedProxies = ConcurrentLinkedQueue() 48 | } 49 | 50 | private val running = Tasks.thread.proxyConnect?.running!! 51 | private val pause = Tasks.thread.proxyConnect?.pause!! 52 | private var retryAttempts = 0 53 | 54 | override fun onApplicationEvent(event : ApplicationReadyEvent) { 55 | if(config.enabledThreads.proxyConnect && !running.get()) 56 | executor.submit { initialize() } 57 | } 58 | 59 | private fun initialize() { 60 | running.set(true) 61 | val supplierProxyListData = SimpleProxyDataList(mutableMapOf()) 62 | //Load proxy from JSON supplier 63 | val proxySupplier = MainProxySupplier(supplierProxyListData, config) 64 | //Load local proxies from "./proxies/*.json" 65 | LocalProxySupplier(supplierProxyListData, config).request().parse() 66 | 67 | //If the supplier is down, skip the supplier and try proxies that only exist within the database 68 | if(retryAttempts < 4) { 69 | try { 70 | proxySupplier 71 | .request() //Requests proxies from the web 72 | .parse() //Attempt to parse the proxies from the web 73 | } catch (e : Exception) { //Attempt to re-run again after 30 seconds if error is captured 74 | retryAttempts++ 75 | logger.error(e.localizedMessage) 76 | Thread.sleep(30000L) 77 | initialize() 78 | return 79 | } 80 | } 81 | retryAttempts = 0 82 | 83 | logger.info("Filtered out the dead proxies, total before: " + supplierProxyListData.proxies.size) 84 | 85 | //Query the database to see if the collected proxies have already been tested and have been dead 86 | //for over 6 months, this reduces the time to test confirmed dead proxies 87 | val proxyIps = mutableListOf() 88 | supplierProxyListData.proxies.mapTo(proxyIps) { it.value.ip } 89 | 90 | val timeNow : LocalDateTime = LocalDateTime.now() 91 | val sixMonthsAgoTimestamp : Timestamp = Timestamp.valueOf(timeNow.minusMonths(6L)) 92 | val oneWeekAgoTimestamp : Timestamp = Timestamp.valueOf(timeNow.minusWeeks(1L)) 93 | val repositoryOfDeadProxiesInLast6Months = repository.findByIpInAndLastSuccessBefore(proxyIps, sixMonthsAgoTimestamp) 94 | val repositoryOfProxiesThatNeverWorked = repository.findByLastSuccessIsNullAndLastTestedBefore(oneWeekAgoTimestamp) 95 | 96 | val deadProxyIps = mutableMapOf() 97 | 98 | //Remove proxies that have not connected within the past 6 months 99 | repositoryOfDeadProxiesInLast6Months 100 | .map { it.ip.toString() } 101 | .forEach { deadProxyIps[it] = it } 102 | 103 | //Removes proxies that have never connected and have been tested for over a week 104 | repositoryOfProxiesThatNeverWorked 105 | .map { it.ip.toString() } 106 | .forEach { deadProxyIps[it] = it } 107 | 108 | supplierProxyListData.proxies 109 | .filter { !deadProxyIps[it.value.ip].isNullOrEmpty() } 110 | .forEach { supplierProxyListData.proxies.remove(it.key) } 111 | 112 | logger.info("One example from the repository: ${repositoryOfDeadProxiesInLast6Months[0].ip}") 113 | logger.info("Filtered out the dead proxies, total after: " + supplierProxyListData.proxies.size) 114 | 115 | //Load proxies that we've already tested successfully, within the database 116 | //(Last successful connection within past 6 months) 117 | DatabaseProxySupplier(supplierProxyListData, repository).request().parse() 118 | 119 | val proxies = Utils.distinctBadIps(supplierProxyListData.proxies.values) 120 | .distinctBy { listOf(it.ip, it.port, it.protocol) } 121 | 122 | logger.info("Loaded ${proxies.size}") 123 | 124 | if(proxies.isNotEmpty()) 125 | prepare(proxies) 126 | } 127 | 128 | //Shuffle proxies & allocate the endpoint server for it to connect to 129 | //We want to shuffle them as some proxies only allow one open connection 130 | //Also try with both Auto Read disabled & enabled 131 | private fun prepare(proxies : List) { 132 | val proxyDataList = mutableListOf() 133 | 134 | for(proxy in proxies) { 135 | val endpointServers = config.endpointServers 136 | for (endpointServer in endpointServers) { 137 | if(endpointServer.name.startsWith("!")) 138 | continue 139 | proxyDataList.add(ProxyChannelData(proxy.ip, proxy.port, proxy.protocol, "", "", 140 | false, endpointServer, ProxyChannelResponseData())) 141 | proxyDataList.add(ProxyChannelData(proxy.ip, proxy.port, proxy.protocol, "", "", 142 | true, endpointServer, ProxyChannelResponseData())) 143 | } 144 | } 145 | //Garbage collect before testing the proxies 146 | System.gc() 147 | 148 | //Shuffled the list so that the proxies are randomized 149 | //This avoids testing the same server on a different port at the same time - 150 | //which can lead to being blocked by the public proxy 151 | val listSize = proxyDataList.size 152 | proxyDataList.shuffled().forEachIndexed { index, proxyData -> 153 | when { //Every 100 proxies, update the checks per second thread 154 | index % 100 == 0 -> { ChecksMonitor.currentIndex = index; ChecksMonitor.proxyListSize = listSize } 155 | } 156 | connect(proxyData) 157 | } 158 | 159 | running.set(false) 160 | 161 | logger.info("Completed ProxyConnect Task") 162 | } 163 | 164 | private fun connect(proxyData : ProxyChannelData) { 165 | val endpoint = proxyData.endpointServer ?: return 166 | val awaitTime = (if(pause.get()) 30000 else config.connectAwait) 167 | 168 | Bootstrap().group(workerGroup) 169 | .channel(NioSocketChannel::class.java) 170 | .resolver(NoopAddressResolverGroup.INSTANCE) 171 | .option(ChannelOption.AUTO_READ, proxyData.autoRead) 172 | .option(ChannelOption.TCP_NODELAY, true) 173 | .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.timeout) 174 | .handler(ProxyChannelInitializer(proxyData)) 175 | .connect(InetSocketAddress(endpoint.ip, endpoint.port)) 176 | .channel().closeFuture().awaitUninterruptibly(awaitTime) 177 | } 178 | 179 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/security/SecurityConfig.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.security 2 | 3 | import org.springframework.context.annotation.Bean 4 | import org.springframework.context.annotation.Configuration 5 | import org.springframework.security.config.Customizer.withDefaults 6 | import org.springframework.security.config.annotation.web.builders.HttpSecurity 7 | import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity 8 | import org.springframework.security.web.SecurityFilterChain 9 | 10 | /** 11 | * SecurityConfig 12 | * 13 | * Allows access to mapped directories, specified in the below functions 14 | * @see filterChain, configure 15 | * 16 | * @author Kai 17 | * @version 1.0, 15/05/2022 18 | */ 19 | @Configuration 20 | @EnableWebSecurity 21 | class SecurityConfig { 22 | 23 | @Bean 24 | fun filterChain(http : HttpSecurity) : SecurityFilterChain { 25 | http.authorizeHttpRequests { 26 | auth -> auth.requestMatchers("*") 27 | .permitAll().anyRequest().authenticated() 28 | }.httpBasic(withDefaults()).csrf().disable() 29 | return http.build() 30 | } 31 | 32 | @Bean 33 | fun configure(http : HttpSecurity): SecurityFilterChain { 34 | http.authorizeHttpRequests { 35 | auth -> auth.requestMatchers("*") 36 | .permitAll().anyRequest().authenticated() 37 | }.httpBasic(withDefaults()) 38 | .csrf().disable() 39 | return http.build() 40 | } 41 | 42 | } 43 | /* Deprecated in 3.0.0 44 | class SecurityConfig : WebSecurityConfigurerAdapter() { 45 | 46 | override fun configure(http : HttpSecurity) { 47 | http.authorizeRequests() 48 | .antMatchers("/*").permitAll() 49 | .antMatchers("/api/v1/*").permitAll() 50 | .anyRequest().authenticated() 51 | //Disable CSRF (TEMP) 52 | //.and().csrf().disable() 53 | .and().csrf().csrfTokenRepository(CookieCsrfTokenRepository.withHttpOnlyFalse()) 54 | } 55 | 56 | }*/ -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/CustomCacheManager.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util 2 | 3 | import org.springframework.cache.CacheManager 4 | import org.springframework.stereotype.Component 5 | import pe.proxy.proxybuilder2.database.EntityForPublicView 6 | 7 | @Component 8 | class CustomCacheManager(private val cache : CacheManager) { 9 | 10 | //Gets the Advanced Proxies from the cache 11 | fun getCachedProxies() : MutableList? { 12 | val cacheEntry = cache.getCache("ProxyController_proxies")?.get("advanced") 13 | val valueWrapper = cacheEntry?.get() as? MutableList<*> 14 | return valueWrapper?.filterIsInstance() as? MutableList 15 | } 16 | 17 | fun getCache() : CacheManager { 18 | return cache 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/KotlinDeserializer.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util 2 | 3 | import kotlinx.serialization.decodeFromString 4 | import kotlinx.serialization.json.Json 5 | import kotlinx.serialization.json.JsonElement 6 | import kotlinx.serialization.json.decodeFromJsonElement 7 | 8 | /** 9 | * KotlinDeserializer 10 | * 11 | * @author Kai 12 | * @version 1.0, 19/05/2022 13 | */ 14 | object KotlinDeserializer { 15 | 16 | val data = Json { this.encodeDefaults = true; this.ignoreUnknownKeys = true } 17 | 18 | inline fun decode(value : String?) : T? { 19 | if(value == null) { 20 | return null 21 | } 22 | return data.decodeFromString(value) 23 | } 24 | 25 | inline fun decodeFromElement(value : JsonElement) : T { 26 | return data.decodeFromJsonElement(value) 27 | } 28 | 29 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/KotlinSerializer.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util 2 | 3 | import kotlinx.serialization.encodeToString 4 | import kotlinx.serialization.json.Json 5 | import kotlinx.serialization.json.JsonElement 6 | import kotlinx.serialization.json.encodeToJsonElement 7 | 8 | /** 9 | * KotlinSerializer 10 | * 11 | * @author Kai 12 | * @version 1.0, 19/05/2022 13 | */ 14 | object KotlinSerializer { 15 | 16 | val json = Json { this.encodeDefaults = true; this.ignoreUnknownKeys = true } 17 | 18 | inline fun encode(value : T) : String { 19 | return json.encodeToString(value) 20 | } 21 | 22 | inline fun encodeElement(value : T) : JsonElement { 23 | return json.encodeToJsonElement(value) 24 | } 25 | 26 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/ProxyConfig.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | 5 | /** 6 | * ProxyConfig 7 | * 8 | * @author Kai 9 | * @version 1.0, 15/05/2022 10 | */ 11 | //@ConstructorBinding <- Deprecated in 3.0.0 12 | @ConfigurationProperties("proxy-config") 13 | data class ProxyConfig(val outputPath : String, val endpointServers : List, 14 | val supplier : ProxySupplier, val proxyCheckIo : ProxyCheckIo, 15 | val connectAwait : Long, val timeout : Int, val githubList : List, 16 | val enabledThreads : EnabledThreads, val twilio : Twilio, 17 | val trustPassword : String, val threads : Int) { 18 | 19 | data class ProxySupplier(val mainUrl : String, val customUrl : String) 20 | 21 | data class EndpointServer(val name : String, val ip : String, val port : Int) 22 | 23 | data class ProxyCheckIo(val apiKey : String) 24 | 25 | data class EnabledThreads(val queryApi : Boolean, val proxyConnect : Boolean, val endpointMonitor : Boolean, 26 | val sqlProxyMonitor: Boolean, val checksPerSecond : Boolean) 27 | 28 | data class Twilio(val sid : String, val token : String, val phoneNumberFrom : String, val phoneNumberTo : String) 29 | 30 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/Tasks.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util 2 | 3 | import org.slf4j.LoggerFactory 4 | import java.util.concurrent.atomic.AtomicBoolean 5 | 6 | /** 7 | * Tasks 8 | * 9 | * @author Kai 10 | * @version 1.0, 29/05/2022 11 | */ 12 | object Tasks { 13 | 14 | private val logger = LoggerFactory.getLogger(Tasks::class.java) 15 | 16 | val thread = RunningThreads() 17 | 18 | data class RunningThreads(val endpointMonitor : RunningThreadAttributes?=RunningThreadAttributes(), 19 | val sqlProxyMonitor: RunningThreadAttributes?=RunningThreadAttributes(), 20 | val proxyConnect : RunningThreadAttributes?=RunningThreadAttributes(), 21 | val queryApi : RunningThreadAttributes?=RunningThreadAttributes(), 22 | val gitActions : RunningThreadAttributes?=RunningThreadAttributes()) { 23 | 24 | fun pauseAllExcept(exemptThread : Any) { 25 | val threads = listOf(endpointMonitor, sqlProxyMonitor, proxyConnect, queryApi, gitActions) 26 | for(thread in threads) 27 | if(thread != exemptThread) 28 | thread?.pause?.set(true) 29 | 30 | logger.info("Pausing all threads except $thread") 31 | } 32 | 33 | fun resumeAll() { 34 | val threads = listOf(endpointMonitor, sqlProxyMonitor, proxyConnect, queryApi, gitActions) 35 | for(thread in threads) 36 | thread?.pause?.set(false) 37 | 38 | logger.info("Resuming all threads") 39 | } 40 | 41 | } 42 | 43 | data class RunningThreadAttributes(val running : AtomicBoolean?=AtomicBoolean(false), 44 | val pause : AtomicBoolean?=AtomicBoolean(false)) 45 | 46 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/Utils.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util 2 | 3 | import com.fasterxml.jackson.core.JsonFactory 4 | import com.fasterxml.jackson.core.JsonToken 5 | import pe.proxy.proxybuilder2.database.ProxyEntity 6 | import pe.proxy.proxybuilder2.net.proxy.data.PerformanceConnectData 7 | import pe.proxy.proxybuilder2.net.proxy.data.SimpleProxyDataType 8 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.LocationData 9 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.OperatorData 10 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.PoliciesData 11 | import pe.proxy.proxybuilder2.net.proxy.proxycheckio.RiskData 12 | import java.sql.Timestamp 13 | import java.time.LocalDateTime 14 | import java.util.* 15 | import java.util.regex.Pattern 16 | import kotlin.reflect.KMutableProperty 17 | import kotlin.reflect.full.memberProperties 18 | 19 | /** 20 | * Utils 21 | * 22 | * Miscellaneous functions used for date/time/math etc 23 | * 24 | * @author Kai 25 | * @version 1.0, 16/05/2022 26 | */ 27 | object Utils { 28 | 29 | val GLOBAL_RANDOM = Random() 30 | val IS_WINDOWS = System.getProperty("os.name").startsWith("Windows") 31 | 32 | fun distinctBadIps(proxies: MutableCollection) : MutableCollection { 33 | val pattern = Pattern.compile("^((\\d|[1-9]\\d|1\\d\\d|2[0-4]\\d|25[0-5])(\\.(?!$)|$)){4}$") 34 | proxies.removeIf { !pattern.matcher(it.ip).matches() } 35 | return proxies 36 | } 37 | 38 | //Checks if proxy ip matches from start to end 39 | fun ipMatch(proxyIp : String, remoteIp : String?) : Boolean = remoteIp != null && remoteIp == proxyIp 40 | 41 | //remoteAddress = /0.0.0.0:8080 42 | fun splitRemoteIp(remoteAddress : String) : String? { 43 | if (!remoteAddress.contains(":")) 44 | return null 45 | return remoteAddress.split(":")[0].replace("/", "") 46 | } 47 | 48 | private fun formatIp(remoteAddress : String): String? { 49 | if(!remoteAddress.contains(":")) 50 | return null 51 | 52 | var ip = "" 53 | val port = remoteAddress.split(":")[1] 54 | 55 | if (remoteAddress.contains(":")) 56 | ip = replaceBadValues( remoteAddress.split(":").toTypedArray()[0]) 57 | 58 | val ipArray = ip.split(".").toTypedArray() 59 | for (i in 0..3) { 60 | if (ipArray[i].startsWith("0") && ipArray[i] != "0") 61 | ipArray[i] = ipArray[i].substring(1) 62 | } 63 | return ipArray.joinToString(separator = ".").plus(":").plus(port) 64 | } 65 | 66 | private fun replaceBadValues(remoteAddress : String) : String = 67 | remoteAddress.replace("\n", "").replace("\r", "") 68 | 69 | fun sortByIp(proxyArray : List) : List { 70 | val comparator : Comparator = Comparator { 71 | ip1, ip2 -> toNumeric(ip1.ip.toString()).compareTo(toNumeric(ip2.ip.toString())) 72 | } 73 | return proxyArray.sortedWith(comparator) 74 | } 75 | 76 | fun sortByIp2(proxyArray : List) : List { 77 | val comparator : Comparator = Comparator { 78 | ip1, ip2 -> toNumeric(ip1).compareTo(toNumeric(ip2)) 79 | } 80 | return proxyArray.sortedWith(comparator) 81 | } 82 | 83 | private fun toNumeric(ip : String) : Long { 84 | if (!ip.contains(".")) 85 | return 0L 86 | 87 | val finalIp = ip.split(".") 88 | 89 | return ((finalIp[0].toLong() shl 24) + (finalIp[1].toLong() shl 16) + 90 | (finalIp[2].toLong() shl 8) + finalIp[3].toLong()) 91 | } 92 | 93 | fun lowestPing(connectionData : PerformanceConnectData) : Long { 94 | // val pingArray = listOf( 95 | // connectionData.aws_NA?.ping, connectionData.ms_HK?.ping, 96 | // connectionData.ora_JP?.ping, connectionData.ora_UK?.ping, 97 | // ) 98 | //Below code fixes bug that causes lowestPing to return null instead of Long 99 | val pingArray = mutableListOf() 100 | connectionData.aws_NA?.ping?.let { pingArray.add(it) } 101 | connectionData.ms_HK?.ping?.let { pingArray.add(it) } 102 | connectionData.ora_JP?.ping?.let { pingArray.add(it) } 103 | connectionData.ora_UK?.ping?.let { pingArray.add(it) } 104 | val pings = pingArray.filter { it != 0L } 105 | return if (pings.isNotEmpty()) { 106 | pings.minOf { it } 107 | } else { 108 | 0L 109 | } 110 | } 111 | 112 | //Custom Serializer - What could go wrong :) 113 | fun serialize(json : String) : List<*> { 114 | val entries = deserialize(json) 115 | 116 | //DO NOT CHANGE THE ORDER -> PoliciesData() has to be last in index 117 | val clazzes = listOf(LocationData(), RiskData(), OperatorData(), PoliciesData()) 118 | 119 | for(clazz in clazzes) run { 120 | clazz::class.memberProperties.filterIsInstance>() 121 | .forEach { 122 | if(entries.containsKey(it.name)) { 123 | val value = entries.getValue(it.name) 124 | it.setter.call(clazz, value) 125 | } 126 | } 127 | } 128 | 129 | return clazzes 130 | } 131 | 132 | //Custom Deserializer - What could go wrong :) 133 | private fun deserialize(json : String) : LinkedHashMap { 134 | val map = LinkedHashMap() 135 | val factory = JsonFactory() 136 | val jsonParser = factory.createParser(json) 137 | 138 | while (!jsonParser.isClosed) { 139 | val nextToken = jsonParser.nextToken() 140 | val name = jsonParser.currentName 141 | var value : Any? = null 142 | 143 | when (nextToken) { 144 | JsonToken.VALUE_STRING -> { 145 | value = jsonParser.valueAsString 146 | //Parse yes & no as true/false Booleans 147 | if(value == "yes" || value == "no") { 148 | value = (value == "yes") 149 | } 150 | } 151 | JsonToken.VALUE_NUMBER_INT -> { 152 | value = if(name == "longitude" || name == "latitude") 153 | jsonParser.floatValue //Happens when longitude is 32 and not 32.841 154 | else 155 | jsonParser.valueAsInt 156 | } 157 | JsonToken.VALUE_TRUE, JsonToken.VALUE_FALSE -> { value = jsonParser.valueAsBoolean } 158 | JsonToken.VALUE_NUMBER_FLOAT -> { value = jsonParser.floatValue } 159 | else -> {} 160 | } 161 | 162 | if(name != null && value != null) 163 | map[name] = value 164 | } 165 | 166 | return map 167 | } 168 | 169 | fun timestampNow() : Timestamp { 170 | return Timestamp.valueOf(LocalDateTime.now()) 171 | } 172 | 173 | fun timestampMinus(minusMinutes : Long) : Timestamp { 174 | return Timestamp.valueOf(LocalDateTime.now().minusMinutes(minusMinutes)) 175 | } 176 | 177 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/writer/CustomFileWriter.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util.writer 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude 4 | import com.fasterxml.jackson.core.JsonFactory 5 | import com.fasterxml.jackson.databind.ObjectMapper 6 | import com.fasterxml.jackson.databind.SerializationFeature 7 | import com.fasterxml.jackson.dataformat.csv.CsvFactory 8 | import com.fasterxml.jackson.dataformat.csv.CsvMapper 9 | import com.fasterxml.jackson.dataformat.csv.CsvSchema 10 | import com.fasterxml.jackson.dataformat.xml.XmlFactory 11 | import com.fasterxml.jackson.dataformat.xml.XmlMapper 12 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 13 | import org.apache.commons.csv.CSVFormat 14 | import org.apache.commons.csv.CSVPrinter 15 | import org.slf4j.LoggerFactory 16 | import pe.proxy.proxybuilder2.database.EntityForPublicView 17 | import pe.proxy.proxybuilder2.database.EntityForPublicViewForCSV 18 | import pe.proxy.proxybuilder2.database.ProxyEntity 19 | import pe.proxy.proxybuilder2.database.ProxyRepository 20 | import pe.proxy.proxybuilder2.net.proxy.data.SupplierProxyListData 21 | import pe.proxy.proxybuilder2.util.ProxyConfig 22 | import pe.proxy.proxybuilder2.util.Utils 23 | import java.io.File 24 | import java.nio.file.Files 25 | 26 | /** 27 | * CustomFileWriter 28 | * 29 | * @author Kai 30 | * @version 1.0, 24/05/2022 31 | */ 32 | class CustomFileWriter(private val repository : ProxyRepository, private val config : ProxyConfig) { 33 | 34 | private val logger = LoggerFactory.getLogger(CustomFileWriter::class.java) 35 | 36 | fun initialize() { 37 | try { 38 | val lastOnlineSince = Utils.timestampMinus(90) //Within the past 90 minutes 39 | val lastOnlineSinceProxies = Utils.sortByIp(repository.findByLastSuccessAfter(lastOnlineSince)) 40 | val archiveProxies = Utils.sortByIp(repository.findByLastSuccessIsNotNull()) 41 | 42 | logger.info("Gathering last online since $lastOnlineSince") 43 | 44 | if (lastOnlineSinceProxies.isEmpty()) 45 | logger.error("Empty proxy list, unable to write") 46 | if (archiveProxies.isEmpty()) 47 | logger.error("Empty proxy archive list, unable to write") 48 | 49 | if(archiveProxies.isEmpty() || lastOnlineSinceProxies.isEmpty()) 50 | return 51 | 52 | val tasks = mutableListOf() 53 | ViewType.values().forEach { viewType -> 54 | when (viewType) { 55 | ViewType.CLASSIC -> { 56 | // val task = Runnable { 57 | val proxies = convertClassic(lastOnlineSinceProxies, viewType) 58 | for (fileExtension in FileExtension.values()) 59 | write(proxies, viewType, fileExtension) 60 | // } 61 | // tasks.add(task) 62 | } 63 | ViewType.ARCHIVE -> { 64 | // val task = Runnable { 65 | val proxies = convert(archiveProxies, viewType) 66 | val classicArchive = convertClassic(archiveProxies, viewType) 67 | for (fileExtension in FileExtension.values()) { 68 | write(proxies, viewType, fileExtension) 69 | write(classicArchive, viewType, fileExtension) 70 | } 71 | // } 72 | // tasks.add(task) 73 | } 74 | ViewType.BASIC, ViewType.ADVANCED -> { 75 | // val task = Runnable { 76 | val proxies = convert(lastOnlineSinceProxies, viewType) 77 | for (fileExtension in FileExtension.values()) 78 | write(proxies, viewType, fileExtension) 79 | // } 80 | //tasks.add(task) 81 | } 82 | } 83 | } 84 | 85 | //Multi-thread the tasks to make it so the application doesn't stall, then wait for the tasks to complete 86 | //tasks.map { executor.submit(it) }.forEach { it.get() } 87 | 88 | logger.info("Completed a heavy task within the CustomFileWriter") 89 | 90 | //Deprecated TODO - Update ReadMe Builder 91 | ReadMeFile(config).create(convert(lastOnlineSinceProxies, ViewType.ADVANCED), archiveProxies) 92 | } catch (e : Exception) { 93 | logger.error(e.localizedMessage) 94 | } 95 | } 96 | 97 | private fun convert(repo : List, viewType: ViewType) : MutableList { 98 | val proxies = mutableListOf() 99 | when (viewType) { 100 | ViewType.BASIC -> { repo.mapTo(proxies) { EntityForPublicView().basic(it) } } 101 | ViewType.ADVANCED -> { repo.mapTo(proxies) { EntityForPublicView().advanced(it) } } 102 | ViewType.ARCHIVE -> { repo.mapTo(proxies) { EntityForPublicView().advanced(it) } } 103 | else -> { } 104 | } 105 | return proxies 106 | } 107 | 108 | private fun convertClassic(repo : List, viewType: ViewType) : SupplierProxyListData { 109 | val proxies = SupplierProxyListData(mutableListOf(), mutableListOf(), mutableListOf(), mutableListOf()) 110 | if (viewType == ViewType.CLASSIC || viewType == ViewType.ARCHIVE) { 111 | repo.forEach { EntityForPublicView().classic(proxies, it) } 112 | } 113 | return proxies 114 | } 115 | 116 | @Throws 117 | private fun write(proxies : List, viewType: ViewType, extension : FileExtension) { 118 | var file = fileBuilder("proxies-${viewType.name.lowercase()}", extension, viewType) 119 | 120 | //Prevent overwriting file within 60 mins 121 | if (file.lastModified() >= Utils.timestampMinus(30).time) 122 | return 123 | 124 | if(extension == FileExtension.TXT && (viewType == ViewType.BASIC || viewType == ViewType.ARCHIVE)) { 125 | //All Proxies 126 | var proxiesAsString = proxies.joinToString(separator = "\n") { "${it.ip}:${it.port}" } 127 | file = fileBuilder("proxies", extension, viewType) 128 | file.writeText(proxiesAsString) 129 | 130 | val protocols = listOf("http", "https", "socks4", "socks5") 131 | for(protocolName in protocols) { 132 | proxiesAsString = proxies.flatMap { prox -> 133 | prox.protocols 134 | ?.map { repo -> prox to repo } 135 | ?.filter { it.second.type == protocolName }!! 136 | }.distinctBy { listOf(it.first.ip, it.second.port) } 137 | .joinToString(separator = "\n") { "${it.first.ip}:${it.second.port}" } 138 | file = fileBuilder("proxies-$protocolName", extension, viewType) 139 | file.writeText(proxiesAsString) 140 | } 141 | return 142 | } 143 | 144 | logger.info("Extension: $extension") 145 | 146 | val mapper = mapper(extension) 147 | ?.setSerializationInclusion(JsonInclude.Include.NON_NULL) 148 | ?.enable(SerializationFeature.INDENT_OUTPUT) 149 | ?: return logger.error("Unable to read mapper extension type") 150 | 151 | if(extension == FileExtension.CSV && mapper is CsvMapper) { 152 | when (viewType) { 153 | ViewType.ADVANCED, ViewType.ARCHIVE -> { 154 | val entity = EntityForPublicViewForCSV().convert(proxies) 155 | val schema: CsvSchema = mapper.schemaFor(EntityForPublicViewForCSV::class.java) 156 | .withHeader().sortedBy(* EntityForPublicViewForCSV().order(viewType)) 157 | mapper.writerFor(List::class.java).with(schema).writeValue(file, entity) 158 | } 159 | ViewType.BASIC -> { } //TODO 160 | else -> { } 161 | } 162 | return 163 | } 164 | 165 | mapper.writeValue(file, proxies) 166 | } 167 | 168 | private fun write(proxies : SupplierProxyListData, viewType: ViewType, extension : FileExtension?) { 169 | if(extension == null) { 170 | return 171 | } 172 | val file = fileBuilder("proxies", extension, viewType) 173 | if (file.lastModified() >= Utils.timestampMinus(30).time) //Prevent overwriting file within 60 mins 174 | return 175 | 176 | val mapper = mapper(extension) 177 | ?.setSerializationInclusion(JsonInclude.Include.NON_NULL) 178 | ?.enable(SerializationFeature.INDENT_OUTPUT) 179 | ?: return logger.error("Unable to read mapper extension type") 180 | 181 | if(extension == FileExtension.CSV && mapper is CsvMapper) { //TODO CHANGE THIS - Proxy Builder 1.0 182 | when (viewType) { 183 | ViewType.CLASSIC, ViewType.ARCHIVE -> { 184 | val writer = Files.newBufferedWriter(file.toPath()) 185 | val format = CSVFormat.Builder.create() 186 | format.setHeader("http", "https", "socks4", "socks5") 187 | 188 | val csvPrinter = CSVPrinter(writer, format.build()) 189 | 190 | val socks4Size = proxies.socks4.size 191 | val socks5Size = proxies.socks5.size 192 | val httpSize = proxies.http.size 193 | val httpsSize = proxies.https.size 194 | 195 | var maxSize = 0 196 | if (socks4Size > maxSize) maxSize = socks4Size 197 | if (socks5Size > maxSize) maxSize = socks5Size 198 | if (httpSize > maxSize) maxSize = httpSize 199 | if (httpsSize > maxSize) maxSize = httpsSize 200 | 201 | for (i in 0 until maxSize) { 202 | var socks4Value = "" 203 | var socks5Value = "" 204 | var httpValue = "" 205 | var httpsValue = "" 206 | if (proxies.socks4.size > i) socks4Value = proxies.socks4[i] 207 | if (proxies.socks5.size > i) socks5Value = proxies.socks5[i] 208 | if (proxies.http.size > i) httpValue = proxies.http[i] 209 | if (proxies.https.size > i) httpsValue = proxies.https[i] 210 | csvPrinter.printRecord(httpValue, httpsValue, socks4Value, socks5Value) 211 | } 212 | csvPrinter.flush() 213 | csvPrinter.close() 214 | } 215 | else -> { } 216 | } 217 | return 218 | } 219 | 220 | mapper.writeValue(file, proxies) 221 | } 222 | 223 | @Throws 224 | fun mapper(extension : FileExtension?) : ObjectMapper? { 225 | if(extension == null) { 226 | return null 227 | } 228 | return when (extension) { 229 | FileExtension.YAML -> { ObjectMapper(YAMLFactory()) } 230 | FileExtension.JSON -> { ObjectMapper(JsonFactory()) } 231 | FileExtension.XML -> { XmlMapper(XmlFactory()) } 232 | FileExtension.CSV -> { CsvMapper(CsvFactory()) } 233 | else -> null 234 | } 235 | } 236 | 237 | private fun fileBuilder(fileName : String, fileExtension : FileExtension, viewType : ViewType) : File { 238 | var filePath = config.outputPath 239 | val extension = fileExtension.name.lowercase() 240 | 241 | //Primary Folder 242 | filePath += (if(viewType == ViewType.ARCHIVE) "/archive/" else "/online-proxies/") 243 | 244 | //Sub Folder 245 | filePath += "$extension/" 246 | 247 | //File Name + Extension 248 | filePath += "$fileName.$extension" 249 | 250 | return File(filePath) //Final Path 251 | } 252 | 253 | enum class ViewType { CLASSIC, BASIC, ADVANCED, ARCHIVE } 254 | 255 | enum class FileExtension { TXT, JSON, YAML, XML, CSV } 256 | 257 | } -------------------------------------------------------------------------------- /src/main/kotlin/pe/proxy/proxybuilder2/util/writer/ReadMeFile.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2.util.writer 2 | 3 | import pe.proxy.proxybuilder2.database.EntityForPublicView 4 | import pe.proxy.proxybuilder2.database.ProxyEntity 5 | import pe.proxy.proxybuilder2.util.ProxyConfig 6 | import java.io.File 7 | import java.text.SimpleDateFormat 8 | import java.util.* 9 | 10 | /** 11 | * ReadMeFile (ProxyBuilder 1.0) 12 | * 13 | * @author Kai 14 | * @version 1.0, 27/07/2021 15 | */ 16 | @Deprecated("Legacy Code - ProxyBuilder 1.0") 17 | class ReadMeFile(private val config : ProxyConfig) { 18 | 19 | private val readmeFile = File("${config.outputPath}/README.md") 20 | 21 | //Legacy code from ProxyBuilder 1.0 - Change in future 22 | fun create(proxies : List, entityArchive : List) { 23 | val readmeText = readmeFile.readText() 24 | 25 | val originalText = readmeText.substring(0, readmeText.indexOf("# [SAMPLE PROXIES]")) 26 | var extraText = 27 | "# [SAMPLE PROXIES] - ${SimpleDateFormat("[MMMM dd yyyy | hh:mm:ss]").format(Date())}\n\n" 28 | 29 | val http = proxies.filter { it.protocols?.any { it1 -> it1.type == "http" } == true } 30 | .map { it.ip + ":" + it.port } 31 | val https = proxies.filter { it.protocols?.any { it1 -> it1.type == "https" } == true } 32 | .map { it.ip + ":" + it.port } 33 | val socks4 = proxies.filter { it.protocols?.any { it1 -> it1.type == "socks4" } == true } 34 | .map { it.ip + ":" + it.port } 35 | val socks5 = proxies.filter { it.protocols?.any { it1 -> it1.type == "socks5" } == true } 36 | .map { it.ip + ":" + it.port } 37 | 38 | val archive = entityArchive.map { it.ip + ":" + it.port } 39 | val uniqueSize = proxies.distinctBy { it.ip }.size 40 | 41 | val proxiesInfoText = 42 | "### Proxy Statistics:\n" + 43 | "- _Online Proxies (By Protocol):_\n" + 44 | " - **SOCKS4** -> ${socks4.size}\n" + 45 | " - **SOCKS5** -> ${socks5.size}\n" + 46 | " - **HTTP** -> ${http.size}\n" + 47 | " - **HTTPS** -> ${https.size}\n\n" + 48 | "- _Proxies (Total):_\n" + 49 | " - **Online Proxies (SOCKS4/5 + HTTP/S)** -> ${proxies.size}\n" + 50 | " - **Unique Online Proxies** -> ${uniqueSize}\n" + 51 | " - **Unique Online/Offline Proxies (Archive)** -> ${archive.size}\n\n" 52 | 53 | //Append proxiesInfoText to extraText 54 | extraText += proxiesInfoText 55 | 56 | val gitHubLinkArray = config.githubList 57 | 58 | val codeTextArray = arrayListOf( 59 | arrayListOf("[SOCKS4 (${socks4.size}/$uniqueSize)](${gitHubLinkArray[0]})", 60 | socks4.take(30).joinToString(separator = "\n")), 61 | arrayListOf("[SOCKS5 (${socks5.size}/$uniqueSize)](${gitHubLinkArray[1]})", 62 | socks5.take(30).joinToString(separator = "\n")), 63 | arrayListOf("[HTTP (${http.size}/$uniqueSize)](${gitHubLinkArray[2]})", 64 | http.take(30).joinToString(separator = "\n")), 65 | arrayListOf("[HTTPS (${https.size}/$uniqueSize)](${gitHubLinkArray[3]})", 66 | https.take(30).joinToString(separator = "\n")), 67 | arrayListOf("[ARCHIVE ($uniqueSize/${archive.size})](${gitHubLinkArray[4]})", 68 | archive.take(30).joinToString(separator = "\n")) 69 | ) 70 | 71 | for(codeText in codeTextArray) { 72 | extraText += ("## ${codeText[0]}" 73 | .plus("\n") 74 | .plus("```yaml") 75 | .plus("\n") 76 | .plus(codeText[1]) 77 | .plus("\n```\n\n" 78 | )) 79 | } 80 | 81 | readmeFile.writeText(originalText 82 | .plus(extraText) 83 | .plus("\n\nThx Co Pure Gs - Sort Meister! \uD83D\uDC9F") 84 | ) 85 | 86 | } 87 | 88 | } -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.profiles.active=dev -------------------------------------------------------------------------------- /src/test/kotlin/pe/proxy/proxybuilder2/MainProxySupplierTest.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2 2 | 3 | import org.junit.jupiter.api.Test 4 | import org.springframework.boot.context.properties.EnableConfigurationProperties 5 | import org.springframework.boot.test.context.SpringBootTest 6 | import pe.proxy.proxybuilder2.util.ProxyConfig 7 | 8 | @SpringBootTest(classes = [ProxyBuilder2Application::class]) 9 | @EnableConfigurationProperties(ProxyConfig::class) 10 | class MainProxySupplierTest { 11 | 12 | @Test 13 | fun getProxyList_from_web_then_parse(apiConfig : ProxyConfig) { 14 | /* val supplierProxyListData = SupplierProxyListData(mutableListOf(), mutableListOf(), mutableListOf(), mutableListOf()) 15 | val mainProxySupplier = MainProxySupplier(supplierProxyListData, apiConfig) 16 | 17 | mainProxySupplier 18 | .request() //Requests proxies from the web 19 | .parse() //Attempt to parse the proxies from the web 20 | 21 | val isProxyListPopulated = !mainProxySupplier.finalProxyList.isEmpty() 22 | 23 | assert(isProxyListPopulated)*/ 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/test/kotlin/pe/proxy/proxybuilder2/ProxyBuilder2ApplicationTests.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2 2 | 3 | import org.junit.jupiter.api.Test 4 | import org.springframework.boot.test.context.SpringBootTest 5 | 6 | @SpringBootTest 7 | class ProxyBuilder2ApplicationTests { 8 | 9 | @Test 10 | fun contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/test/kotlin/pe/proxy/proxybuilder2/SerializerTests.kt: -------------------------------------------------------------------------------- 1 | package pe.proxy.proxybuilder2 2 | 3 | import org.junit.jupiter.api.Test 4 | import org.springframework.boot.test.context.SpringBootTest 5 | 6 | @SpringBootTest 7 | class SerializerTests { 8 | 9 | @Test 10 | fun contextLoads() { 11 | } 12 | 13 | 14 | 15 | } 16 | --------------------------------------------------------------------------------