Prox5
3 |
4 | ### SOCKS5/4/4a validating proxy pool + SOCKS5 server
5 |
6 |
7 |
8 | [](https://pkg.go.dev/git.tcp.direct/kayos/prox5) [](https://goreportcard.com/report/github.com/yunginnanet/prox5) [](ircs://ircd.chat:6697/#tcpdirect) [](https://github.com/yunginnanet/prox5/actions/workflows/go.yml) 
9 |
10 | `import git.tcp.direct/kayos/prox5`
11 |
12 | Prox5 is a golang library for managing, validating, and utilizing a very large amount of arbitrary SOCKS proxies.
13 |
14 | Notably it features interface compatible dialer functions that dial out from different proxies for every connection, and a SOCKS5 server that utilizes those functions.
15 |
16 |
172 |
173 | # **Please see [the docs](https://pkg.go.dev/git.tcp.direct/kayos/prox5) and the [example](example/main.go) for more details.**
174 |
175 |
176 |
--------------------------------------------------------------------------------
/prox5_test.go:
--------------------------------------------------------------------------------
1 | package prox5
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "fmt"
7 | "io"
8 | "net"
9 | "net/http"
10 | "os"
11 | "strconv"
12 | "strings"
13 | "sync"
14 | "sync/atomic"
15 | "testing"
16 | "time"
17 |
18 | "git.tcp.direct/kayos/common/entropy"
19 | "git.tcp.direct/kayos/go-socks5"
20 | )
21 |
22 | var failures = &atomic.Int64{}
23 |
24 | func init() {
25 | _ = os.Setenv("PROX5_SCALER_DEBUG", "1")
26 | failures.Store(0)
27 | }
28 |
29 | type randomFail struct {
30 | t *testing.T
31 | failedCount int64
32 | maxFail int64
33 |
34 | failOneOutOf int
35 | }
36 |
37 | func (rf *randomFail) fail() bool {
38 | if rf.failOneOutOf == 0 {
39 | return false
40 | }
41 |
42 | doFail := entropy.GetOptimizedRand().Intn(rf.failOneOutOf) == 1
43 |
44 | if !doFail {
45 | return false
46 | }
47 | atomic.AddInt64(&rf.failedCount, 1)
48 | rf.t.Logf("random SOCKS failure triggered, total fail count: %d", rf.failedCount)
49 | if rf.maxFail > 0 && atomic.LoadInt64(&rf.failedCount) > rf.maxFail {
50 | rf.t.Errorf("[FAIL] random SOCKS failure triggered too many times, total fail count: %d", rf.failedCount)
51 | }
52 |
53 | failures.Add(1)
54 | return true
55 | }
56 |
57 | type dummyHTTPServer struct {
58 | t *testing.T
59 | net.Listener
60 | }
61 |
62 | func timeNowJSON() []byte {
63 | js, _ := time.Now().MarshalJSON()
64 | return js
65 | }
66 |
67 | func newDummyHTTPSServer(t *testing.T, port int) {
68 | t.Helper()
69 | dtcp := &dummyHTTPServer{t: t}
70 | var err error
71 | if dtcp.Listener, err = net.Listen("tcp", ":"+strconv.Itoa(port)); err != nil && !errors.Is(err, net.ErrClosed) {
72 | t.Fatal(err)
73 | }
74 | go func() {
75 | if err = http.Serve(dtcp, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
76 | time.Sleep(time.Duration(entropy.RNG(300)) * time.Millisecond)
77 | if _, err = w.Write(timeNowJSON()); err != nil {
78 | t.Error("[FAIL] http server failed to write JSON: " + err.Error())
79 | }
80 | })); err != nil && !errors.Is(err, net.ErrClosed) {
81 | t.Error("[FAIL] http.Serve error: " + err.Error())
82 | }
83 | }()
84 |
85 | t.Cleanup(func() {
86 | _ = dtcp.Close()
87 | })
88 |
89 | t.Logf("dummy HTTPS server listening on port %d", port)
90 |
91 | }
92 |
93 | var ErrRandomFail = errors.New("random failure")
94 |
95 | func dummySOCKSServer(t *testing.T, port int, rf ...*randomFail) {
96 | t.Helper()
97 | var failure = &randomFail{t: t, failedCount: int64(0), failOneOutOf: 0}
98 | if len(rf) > 0 {
99 | failure = rf[0]
100 | }
101 |
102 | dialer := func(ctx context.Context, network, addr string) (net.Conn, error) {
103 | if failure.fail() {
104 | return nil, ErrRandomFail
105 | }
106 | time.Sleep(time.Duration(entropy.GetOptimizedRand().Intn(300)) * time.Millisecond)
107 | return net.Dial(network, addr)
108 | }
109 |
110 | server := socks5.NewServer(socks5.WithDial(dialer))
111 | go func() {
112 | err := server.ListenAndServe("tcp", "127.0.0.1:"+strconv.Itoa(port))
113 | if err != nil && !errors.Is(err, net.ErrClosed) {
114 | t.Error("[FAIL] socks server failure: " + err.Error())
115 | }
116 | }()
117 | }
118 |
119 | type p5TestLogger struct {
120 | t *testing.T
121 | }
122 |
123 | func (tl p5TestLogger) Errorf(format string, args ...interface{}) {
124 | tl.t.Logf("[ERROR] "+format, args...)
125 | }
126 | func (tl p5TestLogger) Printf(format string, args ...interface{}) {
127 | val := fmt.Sprintf(format, args...)
128 | if strings.Contains(val, "failed to verify") {
129 | failures.Add(1)
130 | }
131 | tl.t.Logf("[PRINT] " + val)
132 | }
133 | func (tl p5TestLogger) Print(args ...interface{}) {
134 | val := fmt.Sprintf("%+v", args...)
135 | if strings.Contains(val, "failed to verify") {
136 | failures.Add(1)
137 | }
138 | tl.t.Log("[PRINT] " + val)
139 | }
140 | func TestProx5(t *testing.T) {
141 | numTest := 100
142 | if envCount := os.Getenv("PROX5_TEST_COUNT"); envCount != "" {
143 | n, e := strconv.Atoi(envCount)
144 | if e != nil {
145 | t.Skip(e.Error())
146 | }
147 | numTest = n
148 | }
149 | for i := 0; i < numTest; i++ {
150 | dummySOCKSServer(t, 5555+i, &randomFail{
151 | t: t,
152 | failedCount: int64(0),
153 | failOneOutOf: entropy.RNG(200),
154 | maxFail: 50,
155 | })
156 | time.Sleep(time.Millisecond * 5)
157 | }
158 | newDummyHTTPSServer(t, 8055)
159 | time.Sleep(time.Millisecond * 350)
160 | p5 := NewProxyEngine()
161 | p5.SetAndEnableDebugLogger(p5TestLogger{t: t})
162 | p5.SetMaxWorkers(10)
163 | p5.EnableAutoScaler()
164 | p5.SetAutoScalerThreshold(10)
165 | // p5.SetValidationTimeout(200 * time.Millisecond)
166 | p5.SetAutoScalerMaxScale(100)
167 | // p5.DisableRecycling()
168 | p5.SetRemoveAfter(2)
169 | var index = 5555
170 |
171 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
172 | defer cancel()
173 |
174 | var once = &sync.Once{}
175 |
176 | check5 := func() {
177 | if err := p5.Pause(); err != nil {
178 | t.Errorf("[FAIL] failed to pause: %s", err.Error())
179 | }
180 | time.Sleep(time.Second * 1)
181 | got := p5.GetTotalValidated()
182 | want := 55 - failures.Load()
183 | if got != want {
184 | t.Logf("[WARN] total validated proxies does not match expected, got: %d, expected: %d",
185 | got, want)
186 | }
187 | if err := p5.Resume(); err != nil {
188 | t.Errorf("[FAIL] failed to resume: %s", err.Error())
189 | }
190 | }
191 |
192 | load := func() {
193 | if index > 5555+numTest {
194 | return
195 | }
196 | entropy.RandSleepMS(150)
197 | p5.LoadSingleProxy("127.0.0.1:" + strconv.Itoa(index))
198 | if index == 5555+55 {
199 | once.Do(check5)
200 | }
201 | index++
202 | }
203 |
204 | var successCount int64 = 0
205 |
206 | makeReq := func() {
207 | select {
208 | case <-ctx.Done():
209 | return
210 | default:
211 |
212 | }
213 | resp, err := p5.GetHTTPClient().Get("http://127.0.0.1:8055")
214 | if err != nil && !errors.Is(err, ErrNoProxies) && !errors.Is(err, net.ErrClosed) {
215 | t.Error("[FAIL] " + err.Error())
216 | }
217 | if err != nil && errors.Is(err, ErrNoProxies) {
218 | return
219 | }
220 | if resp == nil {
221 | return
222 | }
223 | b, e := io.ReadAll(resp.Body)
224 | if e != nil && !errors.Is(e, net.ErrClosed) {
225 | t.Log("[WARN] " + e.Error())
226 | }
227 | t.Logf("got proxied response: %s", string(b))
228 | atomic.AddInt64(&successCount, 1)
229 | }
230 |
231 | ticker := time.NewTicker(time.Millisecond * 100)
232 |
233 | if err := p5.Start(); err != nil {
234 | t.Fatal(err)
235 | }
236 |
237 | wait := 0
238 |
239 | testLoop:
240 | for {
241 | select {
242 | case <-ctx.Done():
243 | successCountFinal := atomic.LoadInt64(&successCount)
244 | if successCountFinal < 10 {
245 | t.Fatal("no successful requests")
246 | }
247 | t.Logf("total successful requests: %d", successCountFinal)
248 | p5.CloseAllConns()
249 | break testLoop
250 | case <-ticker.C:
251 | // pre-warm
252 | wait++
253 | if wait >= 50 {
254 | go makeReq()
255 | }
256 | default:
257 | load()
258 | }
259 | }
260 | cancel()
261 | if err := p5.Close(); err != nil {
262 | t.Fatal(err)
263 | }
264 | // let the proxy engine close gracefully
265 | time.Sleep(time.Second * 5)
266 | }
267 |
--------------------------------------------------------------------------------
/validator_engine.go:
--------------------------------------------------------------------------------
1 | package prox5
2 |
3 | import (
4 | "bytes"
5 | "crypto/tls"
6 | "errors"
7 | "io"
8 | "net"
9 | "net/http"
10 | "net/url"
11 | "strings"
12 | "sync"
13 | "sync/atomic"
14 | "time"
15 |
16 | "git.tcp.direct/kayos/socks"
17 | "golang.org/x/net/proxy"
18 | )
19 |
20 | var headerPool = sync.Pool{
21 | New: func() interface{} {
22 | hdr := make(http.Header)
23 | hdr["User-Agent"] = []string{""}
24 | hdr["Accept"] = []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"}
25 | hdr["Accept-Language"] = []string{"en-US,en;q=0.5"}
26 | hdr["Accept-Encoding"] = []string{"gzip, deflate, br"}
27 | return hdr
28 | },
29 | }
30 |
31 | func (p5 *ProxyEngine) prepHTTP() (*http.Client, *http.Transport, *http.Request, error) {
32 | req, err := http.NewRequest("GET", p5.GetRandomEndpoint(), bytes.NewBuffer([]byte("")))
33 | if err != nil {
34 | return nil, nil, nil, err
35 | }
36 | headers := headerPool.Get().(http.Header)
37 | headers["User-Agent"] = []string{p5.RandomUserAgent()}
38 |
39 | var client = &http.Client{}
40 | var transporter = &http.Transport{
41 | DisableKeepAlives: true,
42 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
43 | TLSHandshakeTimeout: p5.GetValidationTimeout(),
44 | }
45 |
46 | return client, transporter, req, err
47 | }
48 |
49 | func (sock *Proxy) bad() {
50 | atomic.AddInt64(&sock.timesBad, 1)
51 | }
52 |
53 | func (sock *Proxy) good() {
54 | atomic.AddInt64(&sock.timesValidated, 1)
55 | sock.lastValidated = time.Now()
56 | }
57 |
58 | func httpEndpoint(hmd *handMeDown) (func(r *http.Request) (*url.URL, error), error) {
59 | s := strs.Get()
60 | defer strs.MustPut(s)
61 | s.MustWriteString("http://")
62 | s.MustWriteString(hmd.sock.Endpoint)
63 | purl, err := url.Parse(s.String())
64 | if err != nil {
65 | return nil, err
66 | }
67 | return http.ProxyURL(purl), nil
68 | }
69 |
70 | func (p5 *ProxyEngine) bakeHTTP(hmd *handMeDown) (client *http.Client, req *http.Request, err error) {
71 | builder := strs.Get()
72 | builder.MustWriteString(hmd.protoCheck.String())
73 | builder.MustWriteString("://")
74 | builder.MustWriteString(hmd.sock.Endpoint)
75 | builder.MustWriteString("/?timeout=")
76 | builder.MustWriteString(p5.GetValidationTimeoutStr())
77 | builder.MustWriteString("s")
78 | dialSocks := socks.DialWithConn(builder.String(), hmd.conn)
79 | strs.MustPut(builder)
80 |
81 | var transport *http.Transport
82 |
83 | client, transport, req, err = p5.prepHTTP()
84 | if err != nil {
85 | if req != nil && req.Header != nil {
86 | headerPool.Put(req.Header)
87 | }
88 | return
89 | }
90 |
91 | if hmd.protoCheck != ProtoHTTP {
92 | transport.Dial = dialSocks
93 | client.Transport = transport
94 | return
95 | }
96 |
97 | proxyURL, err := httpEndpoint(hmd)
98 | if err != nil {
99 | if req != nil && req.Header != nil {
100 | headerPool.Put(req.Header)
101 | }
102 | return
103 | }
104 |
105 | transport.Proxy = proxyURL
106 | return
107 | }
108 |
109 | func (p5 *ProxyEngine) validate(hmd *handMeDown) (string, error) {
110 | var (
111 | client *http.Client
112 | req *http.Request
113 | err error
114 | )
115 |
116 | client, req, err = p5.bakeHTTP(hmd)
117 | if err != nil {
118 | return "", err
119 | }
120 |
121 | resp, err := client.Do(req)
122 | defer func() {
123 | if req != nil && req.Header != nil {
124 | headerPool.Put(req.Header)
125 | }
126 | }()
127 | if err != nil {
128 | return "", err
129 | }
130 |
131 | rbody, err := io.ReadAll(resp.Body)
132 | _ = resp.Body.Close()
133 | return string(rbody), err
134 | }
135 |
136 | func (p5 *ProxyEngine) anothaOne() {
137 | p5.stats.Checked.Add(1)
138 | }
139 |
140 | type handMeDown struct {
141 | sock *Proxy
142 | protoCheck ProxyProtocol
143 | conn net.Conn
144 | under proxy.Dialer
145 | }
146 |
147 | func (hmd *handMeDown) Dial(network, addr string) (c net.Conn, err error) {
148 | if hmd.conn.LocalAddr().Network() != network {
149 | return hmd.under.Dial(network, addr)
150 | }
151 | if hmd.conn.RemoteAddr().String() != addr {
152 | return hmd.under.Dial(network, addr)
153 | }
154 | return hmd.conn, nil
155 | }
156 |
157 | func (p5 *ProxyEngine) announceValidating(sock *Proxy, presplit string) {
158 | if sock == nil {
159 | return
160 | }
161 | s := strs.Get()
162 | s.MustWriteString("validating ")
163 | s.MustWriteString(sock.GetProto().String())
164 | s.MustWriteString("://")
165 | s.MustWriteString(presplit)
166 | p5.dbgPrint(s)
167 |
168 | }
169 |
170 | func (p5 *ProxyEngine) singleProxyCheck(sock *Proxy, protocol ProxyProtocol) error {
171 | defer p5.anothaOne()
172 | split := strings.Split(sock.Endpoint, "@")
173 | endpoint := split[0]
174 | if len(split) == 2 {
175 | endpoint = split[1]
176 | }
177 |
178 | // p5.announceValidating(sock, endpoint)
179 |
180 | conn, err := net.DialTimeout("tcp", endpoint, p5.GetValidationTimeout())
181 | if err != nil {
182 | return err
183 | }
184 |
185 | hmd := &handMeDown{sock: sock, conn: conn, under: proxy.Direct, protoCheck: protocol}
186 |
187 | resp, err := p5.validate(hmd)
188 | if err != nil {
189 | p5.badProx.Check(sock)
190 | return err
191 | }
192 |
193 | if newip := net.ParseIP(resp); newip == nil {
194 | p5.badProx.Check(sock)
195 | return errors.New("bad response from http request: " + resp)
196 | }
197 |
198 | sock.ProxiedIP = resp
199 |
200 | return nil
201 | }
202 |
203 | func (sock *Proxy) validate() {
204 | if sock == nil || sock.parent == nil {
205 | return
206 | }
207 | if !atomic.CompareAndSwapUint32(&sock.lock, stateUnlocked, stateLocked) {
208 | return
209 | }
210 | defer atomic.StoreUint32(&sock.lock, stateUnlocked)
211 |
212 | select {
213 | case <-sock.parent.ctx.Done():
214 | return
215 | default:
216 | }
217 |
218 | pe := sock.parent
219 | if pe.useProx.Check(sock) {
220 | // s.dbgPrint("useProx ratelimited: " + sock.Endpoint )
221 | return
222 | }
223 |
224 | // determined as bad, won't try again until it expires from that cache
225 | if pe.badProx.Peek(sock) {
226 | pe.msgBadProxRate(sock)
227 | return
228 | }
229 |
230 | // TODO: consider giving the option for verbose logging of this stuff?
231 |
232 | switch {
233 | case sock.timesValidated == 0, sock.protocol.Get() == ProtoNull:
234 | // try to use the proxy with all 3 SOCKS versions
235 | for tryProto := range protoMap {
236 | if tryProto == ProtoNull {
237 | continue
238 | }
239 | select {
240 | case <-pe.ctx.Done():
241 | return
242 | default:
243 | if err := pe.singleProxyCheck(sock, tryProto); err != nil {
244 | // if the proxy is no good, we continue on to the next.
245 | continue
246 | }
247 | sock.protocol.set(tryProto)
248 | break
249 | }
250 | }
251 | default:
252 | if err := pe.singleProxyCheck(sock, sock.GetProto()); err != nil {
253 | sock.bad()
254 | pe.badProx.Check(sock)
255 | return
256 | }
257 | }
258 |
259 | switch sock.protocol.Get() {
260 | case ProtoSOCKS4, ProtoSOCKS4a, ProtoSOCKS5, ProtoHTTP:
261 | pe.msgChecked(sock, true)
262 | default:
263 | pe.msgChecked(sock, false)
264 | sock.bad()
265 | pe.badProx.Check(sock)
266 | return
267 | }
268 |
269 | sock.good()
270 | pe.tally(sock)
271 | }
272 |
273 | func (p5 *ProxyEngine) tally(sock *Proxy) bool {
274 | var target proxyList
275 | switch sock.protocol.Get() {
276 | case ProtoSOCKS4:
277 | p5.stats.v4()
278 | target = p5.Valids.SOCKS4
279 | case ProtoSOCKS4a:
280 | p5.stats.v4a()
281 | target = p5.Valids.SOCKS4a
282 | case ProtoSOCKS5:
283 | p5.stats.v5()
284 | target = p5.Valids.SOCKS5
285 | case ProtoHTTP:
286 | p5.stats.http()
287 | target = p5.Valids.HTTP
288 | default:
289 | return false
290 | }
291 | target.Lock()
292 | target.PushBack(sock)
293 | target.Unlock()
294 | return true
295 | }
296 |
--------------------------------------------------------------------------------
/setters.go:
--------------------------------------------------------------------------------
1 | package prox5
2 |
3 | import (
4 | "time"
5 |
6 | "git.tcp.direct/kayos/prox5/logger"
7 | )
8 |
9 | // AddUserAgents appends to the list of useragents we randomly choose from during proxied requests
10 | func (p5 *ProxyEngine) AddUserAgents(uagents []string) {
11 | p5.mu.Lock()
12 | p5.opt.userAgents = append(p5.opt.userAgents, uagents...)
13 | p5.mu.Unlock()
14 | p5.DebugLogger.Printf("added %d useragents to cycle through for proxy validation", len(uagents))
15 | }
16 |
17 | // SetUserAgents sets the list of useragents we randomly choose from during proxied requests
18 | func (p5 *ProxyEngine) SetUserAgents(uagents []string) {
19 | p5.mu.Lock()
20 | p5.opt.userAgents = uagents
21 | p5.mu.Unlock()
22 | p5.DebugLogger.Printf("set %d useragents to cycle through for proxy validation", len(uagents))
23 | }
24 |
25 | // SetCheckEndpoints replaces the running list of whatismyip style endpoitns for validation. (must return only the WAN IP)
26 | func (p5 *ProxyEngine) SetCheckEndpoints(newendpoints []string) {
27 | p5.mu.Lock()
28 | p5.opt.checkEndpoints = newendpoints
29 | p5.mu.Unlock()
30 | p5.DebugLogger.Printf("set %d check endpoints for proxy validations", len(newendpoints))
31 | }
32 |
33 | // AddCheckEndpoints appends entries to the running list of whatismyip style endpoitns for validation. (must return only the WAN IP)
34 | func (p5 *ProxyEngine) AddCheckEndpoints(endpoints []string) {
35 | p5.mu.Lock()
36 | p5.opt.checkEndpoints = append(p5.opt.checkEndpoints, endpoints...)
37 | p5.mu.Unlock()
38 | p5.DebugLogger.Printf("added %d check endpoints for proxy validations", len(endpoints))
39 | }
40 |
41 | // SetStaleTime replaces the duration of time after which a proxy will be considered "stale". stale proxies will be skipped upon retrieval.
42 | func (p5 *ProxyEngine) SetStaleTime(newtime time.Duration) {
43 | p5.opt.Lock()
44 | p5.opt.stale = newtime
45 | p5.opt.Unlock()
46 | p5.DebugLogger.Printf("prox5 stale time set to %s", newtime)
47 | }
48 |
49 | // SetValidationTimeout sets the validationTimeout option.
50 | func (p5 *ProxyEngine) SetValidationTimeout(timeout time.Duration) {
51 | p5.opt.Lock()
52 | p5.opt.validationTimeout = timeout
53 | p5.opt.Unlock()
54 | p5.DebugLogger.Printf("prox5 validation timeout set to %s", timeout)
55 | }
56 |
57 | // SetServerTimeout sets the serverTimeout option.
58 | // * serverTimeout defines the timeout for outgoing connections made with the mysteryDialer.
59 | // * To disable timeout on outgoing mysteryDialer connections, set this to time.Duration(0).
60 | func (p5 *ProxyEngine) SetServerTimeout(timeout time.Duration) {
61 | p5.opt.Lock()
62 | p5.opt.serverTimeout = timeout
63 | p5.opt.Unlock()
64 | p5.DebugLogger.Printf("prox5 server timeout set to %s", timeout)
65 | }
66 |
67 | // SetMaxWorkers set the maximum workers for proxy checking.
68 | func (p5 *ProxyEngine) SetMaxWorkers(num int) {
69 | if p5.isEmpty() && num < 2 {
70 | p5.DebugLogger.
71 | Printf("prox5 cannot set max workers to %d, minimum is 2 until we have some valid proxies", num)
72 | num = 2
73 | }
74 | p5.pool.Tune(num)
75 | p5.scaler.SetBaseline(num)
76 |
77 | }
78 |
79 | // EnableRecycling enables recycling used proxies back into the pending channel for revalidation after dispensed.
80 | func (p5 *ProxyEngine) EnableRecycling() {
81 | p5.opt.Lock()
82 | p5.opt.recycle = true
83 | p5.opt.Unlock()
84 | p5.DebugLogger.Printf("prox5 recycling enabled")
85 | }
86 |
87 | // DisableRecycling disables recycling used proxies back into the pending channel for revalidation after dispensed.
88 | func (p5 *ProxyEngine) DisableRecycling() {
89 | p5.opt.Lock()
90 | p5.opt.recycle = false
91 | p5.opt.Unlock()
92 | p5.DebugLogger.Printf("prox5 recycling disabled")
93 | }
94 |
95 | // SetRemoveAfter sets the removeafter policy, the amount of times a recycled proxy is marked as bad before it is removed entirely.
96 | // - Default is 10
97 | // - To disable deleting entirely, set this value to -1
98 | // - Only applies when recycling is enabled
99 | func (p5 *ProxyEngine) SetRemoveAfter(timesfailed int) {
100 | p5.opt.Lock()
101 | p5.opt.removeafter = timesfailed
102 | p5.opt.Unlock()
103 | p5.DebugLogger.Printf("prox5 removeafter policy set to %d", timesfailed)
104 | }
105 |
106 | // SetDialerBailout sets the amount of times the mysteryDialer will dial out and fail before it bails out.
107 | // - The dialer will attempt to redial a destination with a different proxy a specified amount of times before it gives up
108 | func (p5 *ProxyEngine) SetDialerBailout(dialattempts int) {
109 | p5.opt.Lock()
110 | p5.opt.dialerBailout = dialattempts
111 | p5.opt.Unlock()
112 | p5.DebugLogger.Printf("prox5 dialer bailout set to %d", dialattempts)
113 | }
114 |
115 | // SetDispenseMiddleware will add a function that sits within the dialing process of the mysteryDialer and anyhing using it.
116 | // This means this function will be called mid-dial during connections. Return true to approve proxy, false to skip it.
117 | // Take care modiying the proxy in-flight as it is a pointer.
118 | func (p5 *ProxyEngine) SetDispenseMiddleware(f func(*Proxy) (*Proxy, bool)) {
119 | p5.mu.Lock()
120 | p5.dispenseMiddleware = f
121 | p5.mu.Unlock()
122 | p5.DebugLogger.Printf("prox5 dispense middleware set")
123 | }
124 |
125 | // SetDebugLogger sets the debug logger for the ProxyEngine. See the Logger interface for implementation details.
126 | //
127 | // Deprecated: use SetLogger instead. This will be removed in a future version.
128 | func (p5 *ProxyEngine) SetDebugLogger(l logger.Logger) {
129 | p5.SetLogger(l)
130 | }
131 |
132 | // SetLogger sets the debug logger for the ProxyEngine. See the Logger interface for implementation details.
133 | func (p5 *ProxyEngine) SetLogger(l logger.Logger) {
134 | debugHardLock.Lock()
135 | p5.mu.Lock()
136 | p5.DebugLogger = l
137 | p5.mu.Unlock()
138 | debugHardLock.Unlock()
139 | p5.DebugLogger.Printf("prox5 debug logger set")
140 | }
141 |
142 | func (p5 *ProxyEngine) SetAndEnableDebugLogger(l logger.Logger) {
143 | p5.SetLogger(l)
144 | p5.EnableDebug()
145 | }
146 |
147 | // EnableAutoScaler enables the autoscaler.
148 | // This will automatically scale up the number of workers based on the threshold of dial attempts versus validated proxies.
149 | func (p5 *ProxyEngine) EnableAutoScaler() {
150 | p5.scaler.Enable()
151 | p5.DebugLogger.Printf("prox5 autoscaler enabled")
152 | }
153 |
154 | // DisableAutoScaler disables the autoscaler.
155 | func (p5 *ProxyEngine) DisableAutoScaler() {
156 | p5.scaler.Disable()
157 | p5.DebugLogger.Printf("prox5 autoscaler disabled")
158 | }
159 |
160 | // SetAutoScalerMaxScale sets the relative maximum amount that the autoscaler will scale up.
161 | func (p5 *ProxyEngine) SetAutoScalerMaxScale(max int) {
162 | p5.scaler.SetMax(max)
163 | p5.DebugLogger.Printf("prox5 autoscaler max scale set to %d", max)
164 | }
165 |
166 | // SetAutoScalerThreshold sets the threshold of validated proxies versus dials that will trigger the autoscaler.
167 | func (p5 *ProxyEngine) SetAutoScalerThreshold(threshold int) {
168 | p5.scaler.SetThreshold(threshold)
169 | p5.DebugLogger.Printf("prox5 autoscaler threshold set to %d", threshold)
170 | }
171 |
172 | func (p5 *ProxyEngine) EnableDebugRedaction() {
173 | p5.opt.Lock()
174 | p5.opt.redact = true
175 | p5.opt.Unlock()
176 | p5.DebugLogger.Printf("[redacted]")
177 | }
178 |
179 | func (p5 *ProxyEngine) DisableDebugRedaction() {
180 | p5.opt.Lock()
181 | p5.opt.redact = false
182 | p5.opt.Unlock()
183 | p5.DebugLogger.Printf("prox5 redaction disabled")
184 | }
185 |
186 | func (p5 *ProxyEngine) EnableRecyclerShuffling() {
187 | p5.opt.Lock()
188 | p5.opt.shuffle = true
189 | p5.opt.Unlock()
190 | p5.DebugLogger.Printf("prox5 recycler shuffling enabled")
191 | }
192 |
193 | func (p5 *ProxyEngine) DisableRecyclerShuffling() {
194 | p5.opt.Lock()
195 | p5.opt.shuffle = false
196 | p5.opt.Unlock()
197 | p5.DebugLogger.Printf("prox5 recycler shuffling disabled")
198 | }
199 |
200 | func (p5 *ProxyEngine) EnableHTTPClientTLSVerification() {
201 | p5.opt.Lock()
202 | p5.opt.tlsVerify = true
203 | p5.opt.Unlock()
204 | p5.DebugLogger.Printf("prox5 HTTP client TLS verification enabled")
205 | }
206 |
207 | func (p5 *ProxyEngine) DisableHTTPClientTLSVerification() {
208 | p5.opt.Lock()
209 | p5.opt.tlsVerify = false
210 | p5.opt.Unlock()
211 | p5.DebugLogger.Printf("prox5 HTTP client TLS verification disabled")
212 | }
213 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | git.tcp.direct/kayos/common v0.9.7 h1:k2k3fvvEFN9JV+0nyVWLoV8cGRDAhS/8ECO9tEKN+to=
2 | git.tcp.direct/kayos/common v0.9.7/go.mod h1:mmTOIi7k99yygTa1FSOZNoFEEbSTOQV/QpTLUaQU9Tk=
3 | git.tcp.direct/kayos/go-socks5 v0.3.0 h1:nCsYM0ttPZHGAVVG8zFEy2ZTxoSyPp5ld1YSy3zyWDQ=
4 | git.tcp.direct/kayos/go-socks5 v0.3.0/go.mod h1:6Dw8lhiA+dqCY6CvPBtMyXMug9+spSAXC7+lrqItXf0=
5 | git.tcp.direct/kayos/socks v0.1.3 h1:IUogYkM9q+UXEvvAQ5nJJVF03EixLYwbpGnc5FIcH1g=
6 | git.tcp.direct/kayos/socks v0.1.3/go.mod h1:1qQP+wLpoKzINcI7NYRvt7Q0Y5k2dmpe+YtmvlFGNbg=
7 | github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
8 | github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
9 | github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
10 | github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
14 | github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
15 | github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
16 | github.com/gdamore/tcell/v2 v2.7.1 h1:TiCcmpWHiAU7F0rA2I3S2Y4mmLmO9KHxJ7E1QhYzQbc=
17 | github.com/gdamore/tcell/v2 v2.7.1/go.mod h1:dSXtXTSK0VsW1biw65DZLZ2NKr7j0qP/0J7ONmsraWg=
18 | github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
19 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
20 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
21 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
22 | github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI=
23 | github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
24 | github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
25 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
26 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
27 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
28 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
29 | github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
30 | github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
31 | github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
32 | github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
33 | github.com/ooni/oohttp v0.6.7 h1:wmCjx9+gzx7p1xc/kMAmgXSgXKu7G8CAmil4Zii3g10=
34 | github.com/ooni/oohttp v0.6.7/go.mod h1:Vipww76rE6i/Lyd+M8gec/ixPrsyPti1J8xTyqzFIHA=
35 | github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
36 | github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
37 | github.com/panjf2000/ants/v2 v2.9.1 h1:Q5vh5xohbsZXGcD6hhszzGqB7jSSc2/CRr3QKIga8Kw=
38 | github.com/panjf2000/ants/v2 v2.9.1/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
39 | github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
40 | github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
41 | github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
42 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
43 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
44 | github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4=
45 | github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
46 | github.com/refraction-networking/utls v1.6.0 h1:X5vQMqVx7dY7ehxxqkFER/W6DSjy8TMqSItXm8hRDYQ=
47 | github.com/refraction-networking/utls v1.6.0/go.mod h1:kHJ6R9DFFA0WsRgBM35iiDku4O7AqPR6y79iuzW7b10=
48 | github.com/rivo/tview v0.0.0-20230208211350-7dfff1ce7854 h1:/IIOjnKLbuO5YtZUZaJVw9fc062ChPlaGWEBmJ6jyGY=
49 | github.com/rivo/tview v0.0.0-20230208211350-7dfff1ce7854/go.mod h1:lBUy/T5kyMudFzWUH/C2moN+NlU5qF505vzOyINXuUQ=
50 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
51 | github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
52 | github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
53 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
54 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
55 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
56 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
57 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
58 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
59 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
60 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
61 | github.com/yunginnanet/Rate5 v1.3.0 h1:URptaJxOx/TtgddbO6C5f5U+MXSAARN3GYlNYEXdw0Y=
62 | github.com/yunginnanet/Rate5 v1.3.0/go.mod h1:f0r66kVQZojRqUgVdLC/CKexMlF0nUDAmd01tBeF4Ms=
63 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
64 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
65 | golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
66 | golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
67 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
68 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
69 | golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
70 | golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
71 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
72 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
73 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
74 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
75 | golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
76 | golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
77 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
78 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
79 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
80 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
81 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
82 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
83 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
84 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
85 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
86 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
87 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
88 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
89 | golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
90 | golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
91 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
92 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
93 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
94 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
95 | golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
96 | golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
97 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
98 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
99 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
100 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
101 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
102 | golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
103 | golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
104 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
105 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
106 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
107 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
108 | golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
109 | golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
110 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
111 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
112 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
113 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
114 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
115 | nullprogram.com/x/rng v1.1.0 h1:SMU7DHaQSWtKJNTpNFIFt8Wd/KSmOuSDPXrMFp/UMro=
116 | nullprogram.com/x/rng v1.1.0/go.mod h1:glGw6V87vyfawxCzqOABL3WfL95G65az9Z2JZCylCkg=
117 |
--------------------------------------------------------------------------------
/defs.go:
--------------------------------------------------------------------------------
1 | package prox5
2 |
3 | import (
4 | "container/list"
5 | "context"
6 | "sync"
7 | "sync/atomic"
8 | "time"
9 |
10 | "git.tcp.direct/kayos/common/entropy"
11 | cmap "github.com/orcaman/concurrent-map/v2"
12 | "github.com/panjf2000/ants/v2"
13 | rl "github.com/yunginnanet/Rate5"
14 |
15 | "git.tcp.direct/kayos/prox5/internal/scaler"
16 | "git.tcp.direct/kayos/prox5/logger"
17 | )
18 |
19 | type proxyList struct {
20 | *list.List
21 | *sync.RWMutex
22 | }
23 |
24 | func (pl *proxyList) add(p *Proxy) {
25 | pl.Lock()
26 | defer pl.Unlock()
27 | pl.PushBack(p)
28 | }
29 |
30 | func (pl *proxyList) pop() *Proxy {
31 | pl.Lock()
32 | if pl.Len() < 1 {
33 | pl.Unlock()
34 | return nil
35 | }
36 | p := pl.Remove(pl.Front()).(*Proxy)
37 | pl.Unlock()
38 | return p
39 | }
40 |
41 | // ProxyChannels will likely be unexported in the future.
42 | type ProxyChannels struct {
43 | // SOCKS5 is a constant stream of verified SOCKS5 proxies
44 | SOCKS5 proxyList
45 | // SOCKS4 is a constant stream of verified SOCKS4 proxies
46 | SOCKS4 proxyList
47 | // SOCKS4a is a constant stream of verified SOCKS5 proxies
48 | SOCKS4a proxyList
49 | // HTTP is a constant stream of verified SOCKS5 proxies
50 | HTTP proxyList
51 | }
52 |
53 | // Slice returns a slice of all proxyLists in ProxyChannels, note that HTTP is not included.
54 | func (pc ProxyChannels) Slice() []*proxyList {
55 | lists := []*proxyList{&pc.SOCKS5, &pc.SOCKS4, &pc.SOCKS4a}
56 | entropy.GetOptimizedRand().Shuffle(3, func(i, j int) {
57 | lists[i], lists[j] = lists[j], lists[i]
58 | })
59 | return lists
60 | }
61 |
62 | // ProxyEngine represents a proxy pool
63 | type ProxyEngine struct {
64 | Valids ProxyChannels
65 | DebugLogger logger.Logger
66 |
67 | // stats holds the Statistics for ProxyEngine
68 | stats Statistics
69 |
70 | Status uint32
71 |
72 | // Pending is a constant stream of proxy strings to be verified
73 | Pending proxyList
74 |
75 | // see: https://pkg.go.dev/github.com/yunginnanet/Rate5
76 | useProx *rl.Limiter
77 | badProx *rl.Limiter
78 |
79 | dispenseMiddleware func(*Proxy) (*Proxy, bool)
80 |
81 | ctx context.Context
82 | quit context.CancelFunc
83 |
84 | httpOptsDirty *atomic.Bool
85 | httpClients *sync.Pool
86 |
87 | proxyMap *proxyMap
88 |
89 | // reaper sync.Pool
90 |
91 | conKiller chan struct{}
92 |
93 | recycleMu *sync.Mutex
94 | mu *sync.RWMutex
95 | pool *ants.Pool
96 |
97 | scaler *scaler.AutoScaler
98 | scaleTimer *time.Ticker
99 |
100 | recycleTimer *time.Ticker
101 |
102 | lastBadProxAnnnounced *atomic.Value
103 |
104 | opt *config
105 | runningdaemons int32
106 | conductor chan bool
107 | }
108 |
109 | var (
110 | defaultStaleTime = 30 * time.Minute
111 | defaultWorkerCount = 20
112 | defaultBailout = 20
113 | defaultRemoveAfter = 25
114 | // Note: I've chosen to use https here exclusively assuring all validated proxies are SSL capable.
115 | defaultChecks = []string{
116 | "https://wtfismyip.com/text",
117 | "https://myexternalip.com/raw",
118 | "https://ipinfo.io/ip",
119 | "https://api.ipify.org/",
120 | "https://icanhazip.com/",
121 | "https://ifconfig.me/ip",
122 | "https://www.trackip.net/ip",
123 | "https://checkip.amazonaws.com/",
124 | }
125 | )
126 |
127 | // Returns a pointer to our default options (modified and accessed later through concurrent safe getters and setters)
128 | func defOpt() *config {
129 | sm := &config{
130 | useProxConfig: defaultUseProxyRatelimiter,
131 | badProxConfig: defaultBadProxyRateLimiter,
132 |
133 | checkEndpoints: defaultChecks,
134 | userAgents: defaultUserAgents,
135 | RWMutex: &sync.RWMutex{},
136 | removeafter: defaultRemoveAfter,
137 | recycle: true,
138 | debug: true,
139 | dialerBailout: defaultBailout,
140 | stale: defaultStaleTime,
141 | maxWorkers: defaultWorkerCount,
142 | redact: false,
143 | tlsVerify: false,
144 | shuffle: true,
145 | }
146 | sm.validationTimeout = time.Duration(9) * time.Second
147 | sm.serverTimeout = time.Duration(15) * time.Second
148 | return sm
149 | }
150 |
151 | // config holds our configuration for ProxyEngine instances.
152 | // This is implemented as a pointer, and should be interacted with via the setter and getter functions.
153 | type config struct {
154 | // stale is the amount of time since verification that qualifies a proxy going stale.
155 | // if a stale proxy is drawn during the use of our getter functions, it will be skipped.
156 | stale time.Duration
157 | // userAgents contains a list of userAgents to be randomly drawn from for proxied requests, this should be supplied via SetUserAgents
158 | userAgents []string
159 | // debug when enabled will print results as they come in
160 | debug bool
161 | // checkEndpoints includes web services that respond with (just) the WAN IP of the connection for validation purposes
162 | checkEndpoints []string
163 | // maxWorkers determines the maximum amount of workers used for checking proxies
164 | maxWorkers int
165 | // validationTimeout defines the timeout for proxy validation operations.
166 | // This will apply for both the initial quick check (dial), and the second check (HTTP GET).
167 | validationTimeout time.Duration
168 | // serverTimeout defines the timeout for outgoing connections made with the mysteryDialer.
169 | serverTimeout time.Duration
170 | // dialerBailout defines the amount of times a dial atttempt can fail before giving up and returning an error.
171 | dialerBailout int
172 | // redact when enabled will redact the target string from the debug output
173 | redact bool
174 | // recycle determines whether or not we recycle proxies pack into the pending channel after we dispense them
175 | recycle bool
176 | // remove proxy from recycling after being marked bad this many times
177 | removeafter int
178 | // shuffle determines whether or not we shuffle proxies when we recycle them.
179 | shuffle bool
180 | // tlsVerify determines whether or not we verify the TLS certificate of the endpoints the http client connects to.
181 | tlsVerify bool
182 |
183 | // TODO: make getters and setters for these
184 | useProxConfig rl.Policy
185 | badProxConfig rl.Policy
186 |
187 | *sync.RWMutex
188 | }
189 |
190 | // NewDefaultSwamp returns a new ProxyEngine instance.
191 | //
192 | // Deprecated: use NewProxyEngine instead.
193 | func NewDefaultSwamp() *Swamp {
194 | return &Swamp{NewProxyEngine()}
195 | }
196 |
197 | // Swamp is a deprecated alias for ProxyEngine
198 | //
199 | // Deprecated: use ProxyEngine instead.
200 | type Swamp struct {
201 | *ProxyEngine
202 | }
203 |
204 | // NewProxyEngine returns a ProxyEngine with default options.
205 | // After calling this you may use the various "setters" to change the options before calling ProxyEngine.Start().
206 | func NewProxyEngine() *ProxyEngine {
207 | p5 := &ProxyEngine{
208 | stats: Statistics{
209 | birthday: &atomic.Pointer[time.Time]{},
210 | accountingLastDone: &atomic.Pointer[time.Time]{},
211 | },
212 | DebugLogger: &basicPrinter{},
213 |
214 | opt: defOpt(),
215 | lastBadProxAnnnounced: &atomic.Value{},
216 |
217 | conductor: make(chan bool),
218 | mu: &sync.RWMutex{},
219 | recycleMu: &sync.Mutex{},
220 | httpOptsDirty: &atomic.Bool{},
221 | conKiller: make(chan struct{}, 1),
222 | Status: uint32(stateNew),
223 | }
224 |
225 | tnow := time.Now()
226 | p5.stats.birthday.Store(&tnow)
227 | p5.stats.accountingLastDone.Store(&tnow)
228 |
229 | p5.lastBadProxAnnnounced.Store("")
230 | p5.httpOptsDirty.Store(false)
231 | p5.httpClients = &sync.Pool{New: func() interface{} { return p5.newHTTPClient() }}
232 |
233 | stats := []**atomic.Int64{
234 | &p5.stats.Valid4, &p5.stats.Valid4a, &p5.stats.Valid5, &p5.stats.ValidHTTP, &p5.stats.Dispensed,
235 | &p5.stats.Checked, &p5.stats.badAccounted, &p5.stats.Stale,
236 | }
237 | for _, i := range stats {
238 | *i = &atomic.Int64{}
239 | }
240 |
241 | lists := []*proxyList{&p5.Valids.SOCKS5, &p5.Valids.SOCKS4, &p5.Valids.SOCKS4a, &p5.Valids.HTTP, &p5.Pending}
242 | for _, c := range lists {
243 | *c = proxyList{
244 | List: &list.List{},
245 | RWMutex: &sync.RWMutex{},
246 | }
247 | }
248 |
249 | p5.dispenseMiddleware = func(p *Proxy) (*Proxy, bool) {
250 | return p, true
251 | }
252 | p5.ctx, p5.quit = context.WithCancel(context.Background())
253 | // p5.conCtx, p5.killConns = context.WithCancel(context.Background())
254 | p5.proxyMap = newProxyMap(p5)
255 |
256 | atomic.StoreUint32(&p5.Status, uint32(stateNew))
257 | atomic.StoreInt32(&p5.runningdaemons, 0)
258 |
259 | p5.useProx = rl.NewCustomLimiter(p5.opt.useProxConfig)
260 | p5.badProx = rl.NewCustomLimiter(p5.opt.badProxConfig)
261 |
262 | var err error
263 | p5.pool, err = ants.NewPool(p5.opt.maxWorkers, ants.WithOptions(ants.Options{
264 | ExpiryDuration: 2 * time.Minute,
265 | PanicHandler: p5.pondPanic,
266 | }))
267 |
268 | p5.scaler = scaler.NewAutoScaler(p5.opt.maxWorkers, p5.opt.maxWorkers+100, 50)
269 | p5.scaleTimer = time.NewTicker(1 * time.Second)
270 | p5.recycleTimer = time.NewTicker(500 * time.Millisecond)
271 |
272 | if err != nil {
273 | buf := strs.Get()
274 | buf.MustWriteString("CRITICAL: ")
275 | buf.MustWriteString(err.Error())
276 | p5.dbgPrint(buf)
277 | panic(err)
278 | }
279 |
280 | return p5
281 | }
282 |
283 | func newProxyMap(pe *ProxyEngine) *proxyMap {
284 | return &proxyMap{
285 | plot: cmap.New[*Proxy](),
286 | parent: pe,
287 | }
288 | }
289 |
290 | func (p5 *ProxyEngine) pondPanic(p interface{}) {
291 | panic(p)
292 | // pe.dbgPrint("Worker panic: " + fmt.Sprintf("%v", p))
293 | }
294 |
295 | // defaultUserAgents is a small list of user agents to use during validation.
296 | var defaultUserAgents = []string{
297 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
298 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
299 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
300 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/115.0",
301 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
302 | "Mozilla/5.0 (X11; Linux x86_64; rv109.0) Gecko/20100101 Firefox/115.0",
303 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
304 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/115.0",
305 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
306 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.2 Safari/605.1.15",
307 | "Mozilla/5.0 (Windows NT 10.0; rv109.0) Gecko/20100101 Firefox/115.0",
308 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/116.0",
309 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.82",
310 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv109.0) Gecko/20100101 Firefox/115.0",
311 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.1 Safari/605.1.15",
312 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.188",
313 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/114.0",
314 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
315 | "Mozilla/5.0 (X11; Linux x86_64; rv102.0) Gecko/20100101 Firefox/102.0",
316 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.183",
317 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67",
318 | "Mozilla/5.0 (X11; Linux x86_64; rv109.0) Gecko/20100101 Firefox/114.0",
319 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 OPR/100.0.0.0",
320 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv102.0) Gecko/20100101 Firefox/102.0",
321 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/114.0",
322 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.79",
323 | "Mozilla/5.0 (X11; Linux x86_64; rv109.0) Gecko/20100101 Firefox/116.0",
324 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
325 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15",
326 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36",
327 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0",
328 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15",
329 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",
330 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",
331 | "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
332 | "Mozilla/5.0 (Windows NT 10.0; rv102.0) Gecko/20100101 Firefox/102.0",
333 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
334 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15",
335 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
336 | "Mozilla/5.0 (Windows NT 10.0; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5666.197 Safari/537.36",
337 | "Mozilla/5.0 (Windows NT 10.0; rv109.0) Gecko/20100101 Firefox/116.0",
338 | "Mozilla/5.0 (Windows NT 10.0; rv114.0) Gecko/20100101 Firefox/114.0",
339 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv109.0) Gecko/20100101 Firefox/114.0",
340 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
341 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Safari/605.1.15",
342 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
343 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv109.0) Gecko/20100101 Firefox/116.0",
344 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.86",
345 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
346 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv109.0) Gecko/20100101 Firefox/113.0",
347 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
348 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15",
349 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 YaBrowser/23.5.4.674 Yowser/2.5 Safari/537.36",
350 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv109.0) Gecko/20100101 Firefox/116.0",
351 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
352 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
353 | }
354 |
--------------------------------------------------------------------------------