├── README.md ├── main.go ├── LICENSE ├── proxy.go ├── open_proxies.go └── proxies.go /README.md: -------------------------------------------------------------------------------- 1 | # GoProxies 2 | 3 | Collect proxies using Golang; nothing special 4 | 5 | ### Usage 6 | 7 | ``` 8 | go run *.go 9 | ``` 10 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | proxies := newProxyList() 7 | 8 | for i := 0; i < 5; i++ { 9 | p, _ := proxies.getProxy() 10 | fmt.Printf("proxy: %s\ntotal: %d\nindex: %d\nrecent: %d\n\n", p, proxies.count(), proxies._index, len(proxies._proxiesHistory)) 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Mo’ops 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /proxy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | ) 10 | 11 | type Proxy map[string]string 12 | 13 | type Proxies struct { 14 | _proxies []Proxy 15 | _url string 16 | _index int 17 | _ipIndex int 18 | _portIndex int 19 | } 20 | 21 | func newProxies(url string, ipIndex int, portIndex int) *Proxies { 22 | return &Proxies{ 23 | _proxies: []Proxy{}, 24 | _url: url, 25 | _index: 0, 26 | _ipIndex: ipIndex, 27 | _portIndex: portIndex, 28 | } 29 | } 30 | 31 | func (p *Proxies) getProxy() (Proxy, error) { 32 | if p._index >= p.count() { 33 | p._collect() 34 | p._index = 0 35 | } 36 | 37 | if p.count() == 0 { 38 | return Proxy{}, fmt.Errorf("Failed to collect proxies") 39 | } 40 | 41 | proxy := p._proxies[p._index] 42 | p._index++ 43 | 44 | return proxy, nil 45 | } 46 | 47 | func (p Proxies) count() int { 48 | var total int 49 | 50 | for _, element := range p._proxies { 51 | if element != nil { 52 | total++ 53 | } 54 | } 55 | 56 | return total 57 | } 58 | 59 | func (p Proxies) contains(proxy Proxy) bool { 60 | for _, v := range p._proxies { 61 | if v["ip"] == proxy["ip"] && v["port"] == proxy["port"] { 62 | return true 63 | } 64 | } 65 | return false 66 | } 67 | 68 | func (p *Proxies) _collect() { 69 | doc, err := goquery.NewDocument(p._url) 70 | 71 | if err != nil { 72 | log.Fatal(err) 73 | } 74 | 75 | doc.Find("tbody > tr").Each(func(index int, item *goquery.Selection) { 76 | ip := item.Find("td").Eq(p._ipIndex).Text() 77 | port := item.Find("td").Eq(p._portIndex).Text() 78 | proxy := Proxy{"ip": ip, "port": port} 79 | 80 | if strings.Contains(ip, ".") && !p.contains(proxy) { 81 | p._proxies = append(p._proxies, proxy) 82 | } 83 | }) 84 | } 85 | -------------------------------------------------------------------------------- /open_proxies.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "strconv" 10 | ) 11 | 12 | const openProxiesURL = "https://api.openproxy.space/free-proxy-list" 13 | 14 | type respContainer struct { 15 | Proxies []json.RawMessage `json:"proxies"` 16 | } 17 | 18 | type proxyContainer struct { 19 | Ip string `json:"ip"` 20 | Port int `json:"port"` 21 | } 22 | 23 | type OpenProxies struct { 24 | _proxies []Proxy 25 | _index int 26 | _client *http.Client 27 | } 28 | 29 | func newOpenProxies() *OpenProxies { 30 | return &OpenProxies{ 31 | _proxies: []Proxy{}, 32 | _index: 0, 33 | _client: &http.Client{}, 34 | } 35 | } 36 | 37 | func (p OpenProxies) contains(proxy Proxy) bool { 38 | for _, v := range p._proxies { 39 | if v["ip"] == proxy["ip"] && v["port"] == proxy["port"] { 40 | return true 41 | } 42 | } 43 | return false 44 | } 45 | 46 | func (p *OpenProxies) getProxy() (Proxy, error) { 47 | if p._index >= p.count() { 48 | p._collect() 49 | p._index = 0 50 | } 51 | 52 | if p.count() == 0 { 53 | return Proxy{}, fmt.Errorf("Failed to collect proxies") 54 | } 55 | 56 | proxy := p._proxies[p._index] 57 | p._index++ 58 | 59 | return proxy, nil 60 | } 61 | 62 | func (p OpenProxies) count() int { 63 | var total int 64 | 65 | for _, element := range p._proxies { 66 | if element != nil { 67 | total++ 68 | } 69 | } 70 | 71 | return total 72 | } 73 | 74 | func (p *OpenProxies) _collect() { 75 | client := p._client 76 | 77 | // Make GET requests 78 | proxyURL, _ := http.NewRequest("GET", openProxiesURL, nil) 79 | getResp, getErr := client.Do(proxyURL) 80 | 81 | if getErr != nil { 82 | log.Fatalln("Error encounter while making a GET request. ", getErr) 83 | } 84 | 85 | defer getResp.Body.Close() 86 | 87 | body, readErr := ioutil.ReadAll(getResp.Body) 88 | 89 | if readErr != nil { 90 | log.Fatal("Error while posting for the third time. ", readErr) 91 | } 92 | 93 | jsonResp := respContainer{} 94 | err := json.Unmarshal(body, &jsonResp) 95 | 96 | if err != nil { 97 | log.Fatalln("Error unmashalling data. ", err) 98 | } 99 | 100 | for _, k := range jsonResp.Proxies { 101 | _proxy := proxyContainer{} 102 | err2 := json.Unmarshal(k, &_proxy) 103 | 104 | if err2 != nil { 105 | log.Fatalln("Error unmashalling data. ", err2) 106 | } 107 | 108 | proxy := Proxy{"ip": _proxy.Ip, "port": strconv.Itoa(_proxy.Port)} 109 | 110 | if !p.contains(proxy) { 111 | p._proxies = append(p._proxies, proxy) 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /proxies.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "log" 4 | 5 | const ( 6 | // https://www.sslproxies.org/ 7 | sSLProxiesURL = "https://www.sslproxies.org/" 8 | sSLProxiesIPIndex = 0 9 | sSLProxiesPortIndex = 1 10 | 11 | // https://www.socks-proxy.net/ 12 | socksProxiesURL = "https://www.socks-proxy.net/" 13 | socksProxiesIPIndex = 0 14 | socksProxiesPortIndex = 1 15 | 16 | // https://free-proxy-list.net/ 17 | freeProxiesURL = "https://free-proxy-list.net/" 18 | freeProxiesIPIndex = 0 19 | freeProxiesPortIndex = 1 20 | 21 | // Limits 22 | maxHistorySize = 512 23 | ) 24 | 25 | type IProxier interface { 26 | count() int 27 | getProxy() (Proxy, error) 28 | contains(proxy Proxy) bool 29 | } 30 | 31 | func getProxy(p IProxier) (Proxy, error) { 32 | return p.getProxy() 33 | } 34 | 35 | func count(p IProxier) int { 36 | return p.count() 37 | } 38 | 39 | func contains(p IProxier, proxy Proxy) bool { 40 | return p.contains(proxy) 41 | } 42 | 43 | type ProxiesList struct { 44 | _proxiesHistory []Proxy 45 | _proxies []IProxier 46 | _index int 47 | } 48 | 49 | func newProxyList() *ProxiesList { 50 | OpenProxies := newOpenProxies() 51 | SSLProxies := newProxies(sSLProxiesURL, sSLProxiesIPIndex, sSLProxiesPortIndex) 52 | FreeProxies := newProxies(freeProxiesURL, freeProxiesIPIndex, freeProxiesPortIndex) 53 | SocksProxies := newProxies(socksProxiesURL, socksProxiesIPIndex, socksProxiesPortIndex) 54 | 55 | return &ProxiesList{ 56 | _proxies: []IProxier{SSLProxies, SocksProxies, OpenProxies, FreeProxies}, 57 | _proxiesHistory: []Proxy{}, 58 | _index: 0, 59 | } 60 | } 61 | 62 | func (p *ProxiesList) count() int { 63 | var total int 64 | 65 | for _, v := range p._proxies { 66 | total += v.count() 67 | } 68 | 69 | return total 70 | } 71 | 72 | func (p *ProxiesList) getProxy() (Proxy, error) { 73 | 74 | var proxy Proxy 75 | var err error 76 | 77 | defer func() { 78 | if err := recover(); err != nil { 79 | log.Println(err) 80 | } 81 | }() 82 | 83 | for { 84 | if p._index >= len(p._proxies) { 85 | p._index = 0 86 | } 87 | 88 | proxy, err = getProxy(p._proxies[p._index]) 89 | p._index++ 90 | 91 | if err != nil { 92 | log.Fatal(err) 93 | } 94 | 95 | if err == nil && !p._recentProxy(proxy) { 96 | p._addToHistory(proxy) 97 | break 98 | } 99 | } 100 | 101 | return proxy, err 102 | } 103 | 104 | func (p *ProxiesList) _recentProxy(proxy Proxy) bool { 105 | for _, v := range p._proxiesHistory { 106 | if v["ip"] == proxy["ip"] && v["port"] == proxy["port"] { 107 | return true 108 | } 109 | } 110 | return false 111 | } 112 | 113 | func (p *ProxiesList) _addToHistory(proxy Proxy) { 114 | p._proxiesHistory = append(p._proxiesHistory, proxy) 115 | 116 | if len(p._proxiesHistory) >= maxHistorySize { 117 | p._proxiesHistory = p._proxiesHistory[1:] 118 | } 119 | } 120 | 121 | func (p *ProxiesList) expiredProxy(proxy Proxy) { 122 | if !p._recentProxy(proxy) { 123 | p._addToHistory(proxy) 124 | } 125 | } 126 | --------------------------------------------------------------------------------