├── .project ├── .pydevproject ├── .settings └── org.eclipse.core.resources.prefs ├── AutohotKey ├── fileop.ahk ├── hello-pro.ahk ├── hello.ahk ├── mimvp-proxy-autohotkey-curl.ahk ├── mimvp-proxy-autohotkey.ahk └── readme.txt ├── CSharp └── mimvp-proxy-csharp ├── Delphi └── mimvp-proxy-delphi.dpr ├── Go ├── mimvp-proxy-go └── mimvp-proxy-go.go ├── Groovy └── mimvp-proxy-groovy.groovy ├── HTTPie └── mimvp-proxy-httpie.sh ├── Java ├── MimvpProxyJava.java ├── MimvpProxyJava2.java └── lib │ ├── commons-codec-1.10.jar │ ├── commons-logging-1.2.jar │ ├── httpclient-4.5.5.jar │ ├── httpcomponents-client-4.5.5-bin.tar.gz │ ├── httpcomponents-client-4.5.5-src.tar.gz │ └── httpcore-4.4.9.jar ├── NodeJS └── mimvp-proxy-nodejs.js ├── PHP └── mimvp-proxy-php.php ├── Perl └── mimvp-proxy-perl.pl ├── PhantomJS └── mimvp-proxy-phantomjs.js ├── Python2 ├── mimvp-proxy-python2.py ├── socks.py └── socks.pyc ├── Python3 └── mimvp-proxy-python3.py ├── PythonPyspider └── mimvp-proxy-python-pyspider.py ├── PythonRequests ├── mimvp-proxy-python-requests.py ├── socks.py └── socks.pyc ├── PythonScrapy ├── mimvp_proxy_python_scrapy.zip └── mimvp_proxy_python_scrapy │ ├── .project │ ├── .pydevproject │ ├── .settings │ └── org.eclipse.core.resources.prefs │ ├── mimvp_proxy_python_scrapy │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── middlewares.cpython-36.pyc │ │ ├── middlewares2.cpython-36.pyc │ │ └── settings.cpython-36.pyc │ ├── items.py │ ├── middlewares.py │ ├── middlewares2.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ └── mimvp_spider.cpython-36.pyc │ │ └── mimvp_spider.py │ └── scrapy.cfg ├── README.md ├── Ruby └── mimvp-proxy-ruby.rb ├── SeleniumJava ├── MimvpProxy_FirefoxDriver.java └── MimvpProxy_HtmlUnitDriver.java ├── SeleniumPython ├── ghostdriver.log ├── mimvp-proxy-phantomjs.py ├── python_selenium_chrome │ ├── Chrome-proxy-helper │ │ ├── background.js │ │ └── manifest.json │ ├── chrome-proxy-extensions │ │ ├── mimvp-user_mimvp-pass@140.143.62.84_19480.zip │ │ ├── mimvp-user_mimvp-pass@39.106.158.153_14051.zip │ │ └── mimvp-user_mimvp-pass@47.104.234.80_53848.zip │ ├── proxy.zip │ ├── readme.txt │ └── test_selenium_chrome.py └── python_selenium_firefox │ ├── close_proxy_authentication-1.1.xpi │ ├── modify_headers-0.7.1.1-fx.xpi │ └── test_selenium_firefox.py ├── Shell └── mimvp-proxy-shell.sh ├── User-Agent.txt ├── Yiyuyan ├── mimvp-打开网页.e ├── mimvp-易语言基本语法.e ├── mimvp-易语言设置代理.e ├── mimvp-窗体中打开网页.e ├── readme.txt ├── 精易模块.ec ├── 精易模块5.36.ec ├── 超级模块7.3.reg └── 超级模块8.0.ec ├── cssjs ├── eyuyan-demo-1.png ├── eyuyan-demo-2.png ├── jquery.min.js ├── mimvp-logo.png ├── mimvp-proxy-demo-1-lang-proxy-protocol.png ├── mimvp-proxy-demo-2-lang-proxy-demo.png ├── prism-default.css └── prism-default.js └── index.html /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | mimvp-proxy-demo 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | Default 4 | python 2.7 5 | 6 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//Python2/mimvp-proxy-python2.py=utf-8 3 | encoding//Python3/mimvp-proxy-python3.py=utf-8 4 | encoding//PythonRequests/mimvp-proxy-python-requests.py=utf-8 5 | encoding//PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/middlewares.py=utf-8 6 | encoding//PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/middlewares2.py=utf-8 7 | encoding//PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/settings.py=utf-8 8 | encoding//PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/mimvp_spider.py=utf-8 9 | encoding//SeleniumPython/python_selenium_chrome/test_selenium_chrome.py=utf-8 10 | encoding//SeleniumPython/python_selenium_firefox/test_selenium_firefox.py=utf-8 11 | -------------------------------------------------------------------------------- /AutohotKey/fileop.ahk: -------------------------------------------------------------------------------- 1 | ; Example: This is a working script that writes some text to a file then reads it back into memory. 2 | ; It provides the same functionality as this DllCall-example. 3 | 4 | FileSelectFile, FileName, S16,, Create a new file: 5 | if (FileName = "") 6 | return 7 | file := FileOpen(FileName, "w") 8 | if !IsObject(file) 9 | { 10 | MsgBox Can't open "%FileName%" for writing. 11 | return 12 | } 13 | TestString := "This is a test string.`r`n" ; When writing a file this way, use `r`n rather than `n to start a new line. 14 | file.Write(TestString) 15 | file.Close() 16 | 17 | ; Now that the file was written, read its contents back into memory. 18 | file := FileOpen(FileName, "r-d") ; read the file ("r"), share all access except for delete ("-d") 19 | if !IsObject(file) 20 | { 21 | MsgBox Can't open "%FileName%" for reading. 22 | return 23 | } 24 | CharsToRead := StrLen(TestString) 25 | TestString := file.Read(CharsToRead) 26 | file.Close() 27 | MsgBox The following string was read from the file: %TestString% -------------------------------------------------------------------------------- /AutohotKey/hello-pro.ahk: -------------------------------------------------------------------------------- 1 | #0:: 2 | msgbox, 这是我的第一个AutoHotkey脚本 `n 我爱米扑科技 3 | run, http://mimvp.com 4 | return 5 | 6 | #1:: 7 | run, http://proxy.mimvp.com/usercenter/login.php 8 | WinActivate, Chrome ;防止窗口不激活 9 | winwait, 米扑代理 ;等待网页加载成功（至少title显示出来） 10 | sleep, 500 ;保险起见，再等0.5秒（视网速） 11 | send, 'mimvp-user'{tab}'mimvp-pwd'{enter} ;模拟键入米扑代理的登录用户名和密码、回车 12 | return -------------------------------------------------------------------------------- /AutohotKey/hello.ahk: -------------------------------------------------------------------------------- 1 | msgbox, 我的第一个AutoHotkey脚本 `n 我爱米扑科技 2 | run, http://mimvp.com -------------------------------------------------------------------------------- /AutohotKey/mimvp-proxy-autohotkey-curl.ahk: -------------------------------------------------------------------------------- 1 | ; AutohotKey 支持 http 2 | ; 3 | ; 米扑代理示例： 4 | ; http://proxy.mimvp.com/demo2.php 5 | ; 6 | ; 米扑代理购买： 7 | ; http://proxy.mimvp.com 8 | ; 9 | ; mimvp.com 10 | ; 2016-11-29 11 | 12 | URL := "http://proxy.mimvp.com/exist.php" 13 | RESULT_FILE = "result_curl.txt" 14 | 15 | Runcurl = curl.exe "%URL%" -x 138.68.165.154:3128 -o "%RESULT_FILE%" 16 | ;Runcurl = curl.exe "%URL%" -x user:password@proxyhost:port -o "%RESULT_FILE%" 17 | Runwait, %comspec% /c %RunCurl%, , Hide -------------------------------------------------------------------------------- /AutohotKey/mimvp-proxy-autohotkey.ahk: -------------------------------------------------------------------------------- 1 | ; AutohotKey 支持 http 2 | ; 3 | ; 米扑代理示例： 4 | ; http://proxy.mimvp.com/demo2.php 5 | ; 6 | ; 米扑代理购买： 7 | ; http://proxy.mimvp.com 8 | ; 9 | ; mimvp.com 10 | ; 2016-11-29 11 | 12 | 13 | MIMVP_PROXY_NOAUTH := 2 14 | MIMVP_PROXY_AUTH := 1 15 | 16 | ;~ 代理服务器 17 | proxy_http := "138.68.165.154:3128" 18 | 19 | ;~ 要访问的目标页面 20 | mimvp_url := "http://proxy.mimvp.com/exist.php" 21 | mimvp_url2 = "https://proxy.mimvp.com/exist.php" 22 | 23 | whr := ComObjCreate("WinHttp.WinHttpRequest.5.1") 24 | whr.SetTimeouts(30000,30000,30000,30000) ;~ Set timeouts to 30 seconds 25 | 26 | whr.Open("GET", mimvp_url, true) 27 | whr.SetRequestHeader("User-Agent", "curl/7.41.0") ;~ 模拟curl的ua，方便测试 28 | 29 | 30 | ;~ 设置代理服务器 31 | whr.SetProxy(MIMVP_PROXY_NOAUTH, proxy_http) 32 | 33 | ;~ 设置代理隧道验证信息 34 | ;whr.SetCredentials('mimvp-user', 'mimvp-pass', MIMVP_PROXY_AUTH) 35 | 36 | whr.Send() 37 | whr.WaitForResponse() 38 | 39 | MsgBox % whr.ResponseText ; 输入到消息框，网页内容太长则显示不完整 40 | 41 | 42 | ; 打开对话框选择文件，写入完整的网页内容 43 | FileSelectFile, resultName, S16,, Create a new file: 44 | if (resultName = "") 45 | return 46 | 47 | outFile := FileOpen(resultName , "w" , "utf-8") 48 | if !IsObject(outFile) 49 | { 50 | MsgBox , 不能打开文件: %resultName% 51 | return 52 | } 53 | outFile.write(whr.ResponseText) 54 | outFile.Close() 55 | -------------------------------------------------------------------------------- /AutohotKey/readme.txt: -------------------------------------------------------------------------------- 1 | 2 | 本示例的AutoHotkey源码，由米扑科技(mimvp.com)原创提供，转载请注明出处 3 | 4 | 5 | 米扑博客，提供AutoHotkey入门的精品教程： 6 | 7 | AutoHotkey 安装与开发： 8 | http://blog.mimvp.com/2017/09/autohotkey-an-zhuang-yu-kai-fa/ 9 | 10 | 11 | 12 | 13 | 米扑科技： http://mimvp.com 14 | 15 | 米扑代理： http://proxy.mimvp.com 16 | 17 | 十多门语言使用代理的示例代码：http://proxy.mimvp.com/demo2.php#demo-main-div 18 | 19 | 20 | 本示例所有的代理IP，由米扑代理免费提供，感谢！ 21 | 22 | 23 | 24 | 25 | 26 | 米扑代理，提供支持http、https、socks4、socks5的代理IP，覆盖全球120多个国家，中国34个省市，代理质量高 27 | 28 | 买代理，推荐米扑代理： http://proxy.mimvp.com 29 | 30 | 31 | -------------------------------------------------------------------------------- /CSharp/mimvp-proxy-csharp: -------------------------------------------------------------------------------- 1 | /** 2 | C# 支持 http、https 3 | 4 | 米扑代理示例： 5 | https://proxy.mimvp.com/demo2.php 6 | 7 | 米扑代理购买： 8 | https://proxy.mimvp.com 9 | 10 | mimvp.com 11 | 2015-11-09 12 | */ 13 | 14 | using System; 15 | using System.IO; 16 | using System.Net; 17 | using System.Text; 18 | 19 | namespace ConsoleApplication1 20 | { 21 | class Program 22 | { 23 | // 代码级代理 24 | public static void Main(string[] args) 25 | { 26 | System.Net.WebProxy proxy = new WebProxy("218.21.230.156", 4435); // "107.150.96.188", 8080 27 | HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://proxy.mimvp.com/ip.php"); 28 | request.Proxy = proxy; 29 | using (WebResponse response = request.GetResponse()) 30 | { 31 | using (TextReader reader = new StreamReader(response.GetResponseStream())) 32 | { 33 | string line; 34 | while ((line = reader.ReadLine()) != null) 35 | Console.WriteLine(line); 36 | } 37 | } 38 | } 39 | 40 | 41 | // 系统变量级代理 42 | public static void Main2() 43 | { 44 | // your code goes here 45 | System.Net.WebProxy proxy = new WebProxy("107.150.96.188", 8080); 46 | System.Net.HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://proxy.mimvp.com/ip.php"); 47 | request.Proxy = proxy; 48 | request.Timeout = 30 * 1000; 49 | System.Net.HttpWebResponse resp = (HttpWebResponse)request.GetResponse(); 50 | Encoding bin = Encoding.GetEncoding("UTF-8"); 51 | StreamReader reader = new StreamReader(resp.GetResponseStream(), bin); 52 | string str = reader.ReadToEnd(); 53 | Console.WriteLine(str); 54 | 55 | reader.Close(); 56 | reader.Dispose(); 57 | } 58 | 59 | 60 | // 系统变量级代理密码授权 （感谢网友贡献代码：1725242984@qq.com） 61 | public static void Main3() 62 | { 63 | // 要访问的目标页面 64 | string targetUrl = "https://proxy.mimvp.com/ip.php"; 65 | 66 | // 代理ip和隧道验证密码 67 | // 获取用户名密码请见米扑代理 - 会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 68 | string proxyHost = "107.150.96.188"; 69 | string proxyPort = "8080"; 70 | string proxyUser = "mimvp-user"; 71 | string proxyPass = "mimvp-pass"; 72 | 73 | // 设置代理服务器 74 | WebProxy proxy = new WebProxy(string.Format("{0}:{1}", proxyHost, proxyPort), true); 75 | ServicePointManager.Expect100Continue = false; 76 | 77 | var request = WebRequest.Create(targetUrl) as HttpWebRequest; 78 | request.AllowAutoRedirect = true; 79 | request.KeepAlive = true; 80 | request.Method = "GET"; 81 | request.Proxy = proxy; 82 | request.Proxy.Credentials = new System.Net.NetworkCredential(proxyUser, proxyPass); 83 | 84 | using (var response = request.GetResponse() as HttpWebResponse) 85 | using (var sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8)) 86 | { 87 | string htmlStr = sr.ReadToEnd(); 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Delphi/mimvp-proxy-delphi.dpr: -------------------------------------------------------------------------------- 1 | # Delphi 支持 http 2 | # 3 | # 米扑代理示例： 4 | # http://proxy.mimvp.com/demo2.php 5 | # 6 | # 米扑代理购买： 7 | # http://proxy.mimvp.com 8 | # 9 | # mimvp.com 10 | # 2016-11-25 11 | 12 | 13 | interface 14 | uses 15 | Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms, 16 | Dialogs, StdCtrls, IDHTTP; 17 | 18 | type 19 | TForm1 = class(TForm) 20 | Button1: TButton; 21 | procedure Button1Click(Sender: TObject); 22 | private 23 | { Private declarations } 24 | public 25 | { Public declarations } 26 | end; 27 | 28 | var 29 | Form1: TForm1; 30 | 31 | implementation 32 | 33 | {$R * .dfm} 34 | 35 | procedure TForm1.Button1Click(Sender: TObject); 36 | const 37 | // 要访问的目标页面 38 | mimvp_url = "http://proxy.mimvp.com/exist.php" 39 | mimvp_url2 = "https://proxy.mimvp.com/exist.php" 40 | mimvp_url3 = "https://apps.bdimg.com/libs/jquery-i18n/1.1.1/jquery.i18n.min.js" 41 | 42 | // 代理服务器 43 | proxy_ip = '138.68.165.154'; 44 | proxy_port = 3128; 45 | 46 | var 47 | IDHTTP1 : TIDHTTP; 48 | begin 49 | Application.ProcessMessages; 50 | IDHTTP1 : = TIDHTTP.Create(nil); 51 | with IDHTTP1 do 52 | begin 53 | AllowCookies : = True; 54 | HandleRedirects : = True; 55 | ProxyParams.BasicAuthentication : = True; 56 | ProxyParams.ProxyServer : = proxy_ip; 57 | ProxyParams.ProxyPort : = proxy_port; 58 | # ProxyParams.ProxyUsername : = 'mimvp-user'; 59 | # ProxyParams.ProxyPassword : = 'mimvp-pass'; 60 | 61 | Request.Method : = 'GET'; 62 | Request.Accept : = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'; 63 | Request.AcceptEncoding : = 'gzip, deflate, sdch'; 64 | Request.AcceptLanguage : = 'zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4'; 65 | Request.Connection : = 'keep-alive'; 66 | Request.UserAgent : = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'; 67 | IDHTTP1.Get(mimvp_url); 68 | end; 69 | end; 70 | end. -------------------------------------------------------------------------------- /Go/mimvp-proxy-go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Go/mimvp-proxy-go -------------------------------------------------------------------------------- /Go/mimvp-proxy-go.go: -------------------------------------------------------------------------------- 1 | /** 2 | * go 支持 http、socks5 3 | * 4 | * 米扑代理示例： 5 | * http://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * http://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2017.6.20 12 | */ 13 | 14 | package main 15 | 16 | import ( 17 | "fmt" 18 | "io/ioutil" 19 | "log" 20 | "net/http" 21 | "net/url" 22 | "os" 23 | 24 | "golang.org/x/net/proxy" 25 | ) 26 | 27 | // http（方法1） 28 | func main_1(proxy_uri string, mimvp_url string) { 29 | proxy := func(_ *http.Request) (*url.URL, error) { 30 | return url.Parse(proxy_uri) 31 | } 32 | transport := &http.Transport{Proxy: proxy} 33 | client := &http.Client{Transport: transport} 34 | resp, err := client.Get(mimvp_url) 35 | if err != nil { 36 | fmt.Println("error : ", err) 37 | return 38 | } else { 39 | defer resp.Body.Close() 40 | body, _ := ioutil.ReadAll(resp.Body) 41 | fmt.Printf("%s\n", body) 42 | } 43 | } 44 | 45 | // http（方法2） 46 | func main_2(proxy_uri string, mimvp_url string) { 47 | url_i := url.URL{} 48 | url_proxy, _ := url_i.Parse(proxy_uri) 49 | transport := &http.Transport{Proxy: http.ProxyURL(url_proxy)} 50 | client := http.Client{Transport: transport} 51 | resp, err := client.Get(mimvp_url) 52 | if err != nil { 53 | log.Fatalln(err) 54 | } else { 55 | defer resp.Body.Close() 56 | body, _ := ioutil.ReadAll(resp.Body) 57 | fmt.Printf("%s\n", body) 58 | } 59 | } 60 | 61 | // config environment varable 62 | func main_22(proxy_uri string, mimvp_url string) { 63 | // url_i := url.URL{} 64 | // url_proxy, _ := url_i.Parse("https://127.0.0.1:9743") 65 | os.Setenv("HTTP_PROXY", "http://125.77.25.124:80") 66 | os.Setenv("HTTPS_PROXY", "https://210.209.89.100:8081") 67 | c := http.Client{ /* Transport: &http.Transport{ // Proxy: http.ProxyURL(url_proxy)} */ } 68 | resp, err := c.Get("http://mimvp.com") 69 | if err != nil { 70 | log.Fatalln(err) 71 | } else { 72 | defer resp.Body.Close() 73 | body, _ := ioutil.ReadAll(resp.Body) 74 | fmt.Printf("%s\n", body) 75 | } 76 | } 77 | 78 | // 指定代理ip 79 | func getTransportFieldURL(proxy_addr *string) (transport *http.Transport) { 80 | url_i := url.URL{} 81 | url_proxy, _ := url_i.Parse(*proxy_addr) 82 | transport = &http.Transport{Proxy: http.ProxyURL(url_proxy)} 83 | return 84 | } 85 | 86 | // 从环境变量$http_proxy或$HTTP_PROXY中获取HTTP代理地址 87 | // Linux 设置代理环境变量命令： 88 | // http_proxy=http://125.77.25.124:80 89 | // https_proxy=https://210.209.89.100:8081 90 | func getTransportFromEnvironment() (transport *http.Transport) { 91 | transport = &http.Transport{Proxy: http.ProxyFromEnvironment} 92 | return 93 | } 94 | 95 | func fetch(mimvp_url, proxy_uri *string) (html string) { 96 | transport := getTransportFieldURL(proxy_uri) 97 | client := &http.Client{Transport: transport} 98 | req, err := http.NewRequest("GET", *mimvp_url, nil) 99 | req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36") 100 | if err != nil { 101 | log.Fatal(err.Error()) 102 | } 103 | resp, err := client.Do(req) 104 | if err != nil { 105 | log.Fatal(err.Error()) 106 | } 107 | if resp.StatusCode == 200 { 108 | robots, err := ioutil.ReadAll(resp.Body) 109 | resp.Body.Close() 110 | if err != nil { 111 | log.Fatal(err.Error()) 112 | } 113 | html = string(robots) 114 | } else { 115 | html = "" 116 | } 117 | return 118 | } 119 | 120 | // http（方法3） 121 | func main_3(proxy_uri string, mimvp_url string) { 122 | html := fetch(&mimvp_url, &proxy_uri) 123 | fmt.Println(html) 124 | } 125 | 126 | // socks5 127 | // 1. download https://github.com/golang/net 128 | // 2. unzip and move to /usr/local/go/src/golang.org/x/net/ 129 | // 3. import ( "golang.org/x/net/proxy" ) 130 | func main_socks5(proxy_socks string, mimvp_url string) { 131 | dialer, err := proxy.SOCKS5("tcp", proxy_socks, nil, proxy.Direct) 132 | if err != nil { 133 | fmt.Println(os.Stderr, "can't connect to socks5 proxy:", err) 134 | os.Exit(1) 135 | } 136 | 137 | // setup a http client 138 | // transport := &http.Transport{} 139 | // transport.Dial = dialer.Dial 140 | transport := &http.Transport{Dial: dialer.Dial} 141 | client := &http.Client{Transport: transport} 142 | 143 | resp, err := client.Get(mimvp_url) 144 | if err != nil { 145 | log.Fatalln(err) 146 | } else { 147 | defer resp.Body.Close() 148 | body, _ := ioutil.ReadAll(resp.Body) 149 | fmt.Printf("%s\n", body) 150 | } 151 | } 152 | 153 | func main() { 154 | // http 155 | proxy_uri := "http://125.77.25.124:80" 156 | mimvp_url := "http://proxy.mimvp.com/exist.php" 157 | main_1(proxy_uri, mimvp_url) 158 | main_2(proxy_uri, mimvp_url) 159 | main_3(proxy_uri, mimvp_url) 160 | 161 | // socks5 162 | proxy_socks := "175.138.65.244:1080" 163 | main_socks5(proxy_socks, mimvp_url) 164 | } 165 | 166 | // 执行命令： 167 | // $ go build mimvp-proxy-go.go 168 | // $ ./mimvp-proxy-go 169 | -------------------------------------------------------------------------------- /Groovy/mimvp-proxy-groovy.groovy: -------------------------------------------------------------------------------- 1 | /** 2 | * Groovy 支持 http 3 | * 4 | * 米扑代理示例： 5 | * http://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * http://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2017-07-18 12 | * 13 | * Groovy 爬取网页示例： 14 | * http://blog.mimvp.com/2017/09/groovy-tong-guo-dai-li-zhua-qu-wang-ye/ 15 | */ 16 | 17 | 18 | @Grab(group='org.codehaus.groovy.modules.http-builder', module='http-builder', version='0.7.1' ) 19 | 20 | import groovyx.net.http.HTTPBuilder 21 | import static groovyx.net.http.ContentType.* 22 | import static groovyx.net.http.Method.* 23 | import org.apache.http.auth.* 24 | 25 | 26 | class MimvpSpider { 27 | static def proxy_http = "http://208.92.93.218:1080" 28 | 29 | static def mimvp_url = "http://proxy.mimvp.com/exist.php" 30 | static def mimvp_url2 = "https://proxy.mimvp.com/exist.php" 31 | 32 | 33 | static main(args) { 34 | spider_proxy(MimvpSpider.mimvp_url, MimvpSpider.proxy_http) // http 35 | spider_proxy(MimvpSpider.mimvp_url2, MimvpSpider.proxy_http) // https 36 | } 37 | 38 | 39 | static spider_proxy(mimvp_url, proxy) { 40 | def http = new HTTPBuilder(mimvp_url) 41 | 42 | // http.client.getCredentialsProvider().setCredentials( 43 | // new AuthScope("myproxy.com", 8080), 44 | // new UsernamePasswordCredentials("proxy-username", "proxy-password") 45 | // ) 46 | 47 | def proxy_type = proxy.split("://")[0] 48 | def proxy_ip = proxy.split("://")[1].split(":")[0] 49 | def proxy_port = proxy.split("://")[1].split(":")[1] 50 | proxy_port = proxy_port.toInteger() 51 | 52 | println proxy_type 53 | println proxy_ip 54 | println proxy_port 55 | 56 | 57 | // http.setProxy('myproxy.com', 8080, 'http') 58 | http.setProxy(proxy_ip, proxy_port, proxy_type) 59 | http.ignoreSSLIssues() // 访问https网站，不需验证， module='http-builder', version='0.7.1' 60 | 61 | http.request( GET, TEXT ){ req -> 62 | response.success = { resp, reader -> 63 | println "Response: ${reader.text}" 64 | } 65 | } 66 | } 67 | } 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /HTTPie/mimvp-proxy-httpie.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # http 支持 http、https 4 | # 5 | # 米扑代理示例： 6 | # http://proxy.mimvp.com/demo2.php 7 | # 8 | # 米扑代理购买： 9 | # http://proxy.mimvp.com 10 | # 11 | # mimvp.com 12 | # 2015-11-10 13 | 14 | 15 | # http代理格式 --proxy http:IP:Port 16 | # https代理格式 --proxy https:IP:Port 17 | 18 | 19 | $ http --help 20 | usage: http [--json] [--form] [--pretty {all,colors,format,none}] 21 | [--style STYLE] [--print WHAT] [--headers] [--body] [--verbose] 22 | [--all] [--history-print WHAT] [--stream] [--output FILE] 23 | [--download] [--continue] 24 | [--session SESSION_NAME_OR_PATH | --session-read-only SESSION_NAME_OR_PATH] 25 | [--auth USER[:PASS]] [--auth-type {basic,digest}] 26 | [--proxy PROTOCOL:PROXY_URL] [--follow] 27 | [--max-redirects MAX_REDIRECTS] [--timeout SECONDS] 28 | [--check-status] [--verify VERIFY] 29 | [--ssl {ssl2.3,ssl3,tls1,tls1.1,tls1.2}] [--cert CERT] 30 | [--cert-key CERT_KEY] [--ignore-stdin] [--help] [--version] 31 | [--traceback] [--default-scheme DEFAULT_SCHEME] [--debug] 32 | [METHOD] URL [REQUEST_ITEM [REQUEST_ITEM ...]] 33 | 34 | # http 35 | http --proxy "http:217.107.197.174:8081" http://proxy.mimvp.com/exist.php | grep '' 36 | 37 | # https 38 | http --verify no --proxy "https:46.105.214.133:3128" https://proxy.mimvp.com/exist.php | grep '' 39 | -------------------------------------------------------------------------------- /Java/MimvpProxyJava.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Java 支持 http、https、socks4、socks5 3 | * 4 | * 米扑代理示例： 5 | * https://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * https://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2015-11-09 12 | */ 13 | 14 | 15 | package com.mimvp; 16 | 17 | import java.io.IOException; 18 | import java.io.InputStream; 19 | import java.io.InputStreamReader; 20 | import java.net.Authenticator; 21 | import java.net.InetSocketAddress; 22 | import java.net.PasswordAuthentication; 23 | import java.net.Proxy; 24 | import java.net.Socket; 25 | import java.net.URL; 26 | import java.net.URLConnection; 27 | import java.security.KeyManagementException; 28 | import java.security.NoSuchAlgorithmException; 29 | import java.security.cert.X509Certificate; 30 | import java.util.HashMap; 31 | import java.util.Iterator; 32 | import java.util.Properties; 33 | 34 | import javax.net.ssl.HostnameVerifier; 35 | import javax.net.ssl.HttpsURLConnection; 36 | import javax.net.ssl.SSLContext; 37 | import javax.net.ssl.SSLSession; 38 | import javax.net.ssl.TrustManager; 39 | import javax.net.ssl.X509TrustManager; 40 | 41 | 42 | public class MimvpProxyJava { 43 | 44 | @SuppressWarnings({ "serial" }) 45 | public static HashMap proxyMap = new HashMap() { 46 | { 47 | put("http", "138.68.161.14:3128"); 48 | put("https", "61.216.1.23:3128"); 49 | put("socks4", "115.238.247.205:1080"); 50 | put("socks5", "122.146.58.135:16405"); 51 | } 52 | }; 53 | 54 | 55 | public static String PROXY_USERNAME = "username"; 56 | public static String PROXY_PASSWORD = "password"; 57 | @SuppressWarnings({ "serial" }) 58 | public static HashMap proxyAuthMap = new HashMap() { 59 | { 60 | put("http", "120.24.177.37:6474"); 61 | put("https", "120.24.177.37:6474"); 62 | put("socks5", "120.24.177.37:6476"); 63 | } 64 | }; 65 | 66 | 67 | final static String proxyUrl = "http://proxy.mimvp.com/test_proxy2.php"; // http 68 | final static String proxyUrl2 = "https://proxy.mimvp.com/test_proxy2.php"; // https 69 | 70 | 71 | // 全局禁止ssl证书验证，防止访问非验证的https网址无法访问，例如：https://mimvp.com 72 | static { 73 | disableSslVerification(); 74 | } 75 | 76 | private static void disableSslVerification() { 77 | try { 78 | // Create a trust manager that does not validate certificate chains 79 | TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { 80 | public java.security.cert.X509Certificate[] getAcceptedIssuers() { 81 | return null; 82 | } 83 | public void checkClientTrusted(X509Certificate[] certs, String authType) { 84 | } 85 | public void checkServerTrusted(X509Certificate[] certs, String authType) { 86 | } 87 | }}; 88 | 89 | // Install the all-trusting trust manager 90 | SSLContext sc = SSLContext.getInstance("SSL"); 91 | sc.init(null, trustAllCerts, new java.security.SecureRandom()); 92 | HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); 93 | 94 | // Create all-trusting host name verifier 95 | HostnameVerifier allHostsValid = new HostnameVerifier() { 96 | public boolean verify(String hostname, SSLSession session) { 97 | return true; 98 | } 99 | }; 100 | 101 | // Install the all-trusting host verifier 102 | HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); 103 | } catch (NoSuchAlgorithmException e) { 104 | e.printStackTrace(); 105 | } catch (KeyManagementException e) { 106 | e.printStackTrace(); 107 | } 108 | } 109 | 110 | 111 | // 主入口函数 112 | public static void main(String args[]){ 113 | spider_proxy(); // 通过API实时获取米扑代理 114 | proxy_no_auth(); // 代理无用户名密码授权 115 | proxy_auth(); // 代理需要用户名密码授权 116 | } 117 | 118 | 119 | // 通过API实时获取米扑代理 120 | public static void spider_proxy() { 121 | String proxy_url = "https://proxyapi.mimvp.com/api/fetchsecret.php?orderid=868435221231212345&http_type=3"; 122 | 123 | URL url = null; 124 | try { 125 | url = new URL(proxy_url); 126 | 127 | URLConnection conn = url.openConnection(); 128 | conn.setDoOutput(true); 129 | conn.setReadTimeout(30 * 1000); // 设置超时30秒 130 | conn.setConnectTimeout(30 * 1000); // 连接超时30秒 131 | 132 | InputStream in = conn.getInputStream(); 133 | InputStreamReader reader = new InputStreamReader(in); 134 | char[] ch = new char[1024]; 135 | int len = 0; 136 | String data = ""; 137 | while((len = reader.read(ch)) > 0) { 138 | String newData = new String(ch, 0, len); 139 | data += newData; 140 | } 141 | 142 | String[] proxy_list; 143 | proxy_list = data.split("\n"); 144 | for (String proxy : proxy_list) { 145 | System.out.println(proxy); 146 | } 147 | in.close(); 148 | } catch (Exception e) { 149 | e.printStackTrace(); 150 | } 151 | } 152 | 153 | 154 | // 使用代理，无用户名密码授权 155 | public static void proxy_no_auth() { 156 | int dataLen = 0; 157 | 158 | // proxy protocol 只支持 http、socks5 159 | dataLen = proxy_protocol("http", MimvpProxyJava.proxyMap.get("http")); // http 160 | System.out.println("http : " + MimvpProxyJava.proxyMap.get("http") + " --> " + dataLen); 161 | 162 | dataLen = proxy_protocol("socks5", MimvpProxyJava.proxyMap.get("socks5")); // socks5 163 | System.out.println("socks5 : " + MimvpProxyJava.proxyMap.get("socks5") + " --> " + dataLen); 164 | 165 | 166 | // proxy property 支持http、https、socks4、socks5 167 | Iterator it2 = MimvpProxyJava.proxyMap.keySet().iterator(); 168 | while(it2.hasNext()){ 169 | String proxyType = it2.next(); 170 | String proxyIpPort = MimvpProxyJava.proxyMap.get(proxyType); 171 | dataLen = proxy_property(proxyType, proxyIpPort); 172 | System.out.println(proxyType + " : " + proxyIpPort + " --> " + dataLen); 173 | } 174 | 175 | 176 | // proxy socks 177 | Iterator it3 = MimvpProxyJava.proxyMap.keySet().iterator(); 178 | while(it3.hasNext()){ 179 | String proxyType = it3.next(); 180 | String proxyIpPort = MimvpProxyJava.proxyMap.get(proxyType); 181 | dataLen = proxy_socks(proxyType, proxyIpPort); 182 | System.out.println(proxyType + " : " + proxyIpPort + " --> " + dataLen); 183 | } 184 | } 185 | 186 | 187 | // 使用代理，需要用户名密码授权，请先取消授权的注释（代码里有注释说明） 188 | public static void proxy_auth() { 189 | int dataLen = 0; 190 | 191 | // proxy protocol 只支持 http、socks5 192 | dataLen = proxy_protocol("http", MimvpProxyJava.proxyAuthMap.get("http")); // http 193 | System.out.println("http : " + MimvpProxyJava.proxyAuthMap.get("http") + " --> " + dataLen); 194 | 195 | dataLen = proxy_protocol("socks5", MimvpProxyJava.proxyAuthMap.get("socks5")); // socks5 196 | System.out.println("socks5 : " + MimvpProxyJava.proxyAuthMap.get("socks5") + " --> " + dataLen); 197 | 198 | 199 | // proxy property 支持http、https、socks4、socks5 200 | Iterator it2 = MimvpProxyJava.proxyAuthMap.keySet().iterator(); 201 | while(it2.hasNext()){ 202 | String proxyType = it2.next(); 203 | String proxyIpPort = MimvpProxyJava.proxyAuthMap.get(proxyType); 204 | dataLen = proxy_property(proxyType, proxyIpPort); 205 | System.out.println(proxyType + " : " + proxyIpPort + " --> " + dataLen); 206 | } 207 | } 208 | 209 | 210 | // 设置系统代理，支持全部协议 http，https，socks4，socks5 211 | private static int proxy_property(String proxyType, String proxyIpPort) { 212 | int dataLen = 0; 213 | 214 | String proxy_ip = proxyIpPort.split(":")[0]; 215 | String proxy_port = proxyIpPort.split(":")[1]; 216 | 217 | Properties prop = System.getProperties(); 218 | 219 | // http 220 | if(proxyType.equals("http")){ 221 | prop.setProperty("http.proxySet", "true"); 222 | prop.setProperty("http.proxyHost", proxy_ip); 223 | prop.setProperty("http.proxyPort", proxy_port); 224 | prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*"); 225 | } 226 | 227 | // https 228 | if (proxyType.equals("https")) { 229 | prop.setProperty("https.proxyHost", proxy_ip); 230 | prop.setProperty("https.proxyPort", proxy_port); 231 | } 232 | 233 | // 支持同时代理 http和https 请求 234 | if (proxyType.equals("http") || proxyType.equals("https")) { 235 | prop.setProperty("proxyHost", proxy_ip); 236 | prop.setProperty("proxyPort", proxy_port); 237 | } 238 | 239 | // socks 240 | if(proxyType.equals("socks4") || proxyType.equals("socks5")){ 241 | prop.setProperty("socksProxySet", "true"); 242 | prop.setProperty("socksProxyHost", proxy_ip); 243 | prop.setProperty("socksProxyPort", proxy_port); 244 | } 245 | 246 | // ftp 247 | if(proxyType.equals("ftp")){ 248 | prop.setProperty("ftp.proxyHost", proxy_ip); 249 | prop.setProperty("ftp.proxyPort", proxy_port); 250 | prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*"); 251 | } 252 | 253 | // // auth 代理需要用户名和密码授权时开启，取消此注释，米扑代理验证通过 254 | // Authenticator.setDefault(new MyAuthenticator(MimvpProxyJava.PROXY_USERNAME, MimvpProxyJava.PROXY_PASSWORD)); 255 | 256 | try{ 257 | URL url = new URL(proxyUrl); // 默认访问http网页 258 | if(proxyType.equals("https")) { // 若为https协议，则访问https网页 259 | url = new URL(proxyUrl2); 260 | } 261 | URLConnection conn = url.openConnection(); 262 | conn.setConnectTimeout(30 * 1000); 263 | 264 | InputStream in = conn.getInputStream(); 265 | InputStreamReader reader = new InputStreamReader(in); 266 | char[] ch = new char[1024]; 267 | int len = 0; 268 | String data = ""; 269 | while((len = reader.read(ch)) > 0) { 270 | String newData = new String(ch, 0, len); 271 | data += newData; 272 | } 273 | in.close(); 274 | 275 | System.out.println("data : " + data); 276 | dataLen = data.length(); 277 | } catch(Exception e) { 278 | e.printStackTrace(); 279 | } 280 | prop.clear(); // 清空属性参数，防止下次设置代理时残留脏数据 281 | prop = null; 282 | 283 | return dataLen; 284 | } 285 | 286 | static class MyAuthenticator extends Authenticator { 287 | private String username = ""; 288 | private String password = ""; 289 | public MyAuthenticator(String username, String password) { 290 | this.username = username; 291 | this.password = password; 292 | } 293 | protected PasswordAuthentication getPasswordAuthentication() { 294 | return new PasswordAuthentication(this.username, this.password.toCharArray()); 295 | } 296 | } 297 | 298 | 299 | // 使用函数协议，仅支持 HTTP 和 SOCKS5 300 | private static int proxy_protocol(String proxyType, String proxyIpPort) { 301 | int dataLen = 0; 302 | 303 | String proxy_ip = proxyIpPort.split(":")[0]; 304 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 305 | 306 | try{ 307 | URL url = new URL(proxyUrl); // http://proxy.mimvp.com 308 | 309 | // // auth 代理需要用户名密码授权时开启，取消此注释，米扑代理验证通过 310 | // Authenticator.setDefault(new MyAuthenticator(MimvpProxyJava.PROXY_USERNAME, MimvpProxyJava.PROXY_PASSWORD)); 311 | 312 | InetSocketAddress addr = new InetSocketAddress(proxy_ip, proxy_port); 313 | Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); 314 | if(proxyType.equals("socks4") || proxyType.equals("socks5")) { 315 | proxy = new Proxy(Proxy.Type.SOCKS, addr); 316 | } 317 | 318 | URLConnection conn = url.openConnection(proxy); 319 | conn.setConnectTimeout(30 * 1000); // 连接超时30秒 320 | 321 | InputStream in = conn.getInputStream(); 322 | InputStreamReader reader = new InputStreamReader(in); 323 | char[] ch = new char[1024]; 324 | int len = 0; 325 | String data = ""; 326 | while((len = reader.read(ch)) > 0) { 327 | String newData = new String(ch, 0, len); 328 | data += newData; 329 | } 330 | in.close(); 331 | 332 | System.out.println("data : " + data); 333 | dataLen = data.length(); 334 | } catch(Exception e) { 335 | e.printStackTrace(); 336 | } 337 | return dataLen; 338 | } 339 | 340 | 341 | // proxy socket，测试用 342 | private static int proxy_socks(String proxyType, String proxyIpPort) { 343 | int dataLen = 0; 344 | Socket socket = null; 345 | 346 | String proxy_ip = proxyIpPort.split(":")[0]; 347 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 348 | 349 | try { 350 | socket = new Socket(proxy_ip, proxy_port); 351 | 352 | byte[] ch = new String("GET http://www.mimvp.com/ HTTP/1.1\r\n\r\n").getBytes(); 353 | socket.getOutputStream().write(ch); 354 | socket.setSoTimeout(30 * 1000); 355 | 356 | byte[] bt = new byte[1024]; 357 | InputStream in = socket.getInputStream(); 358 | int len = 0; 359 | String data = ""; 360 | while((len = in.read(bt)) > 0) { 361 | String newData = new String(bt, 0, len); 362 | data += newData; 363 | } 364 | System.out.println("data : " + data); 365 | dataLen = data.length(); 366 | }catch(Exception e) { 367 | e.printStackTrace(); 368 | } finally{ 369 | try { 370 | if(socket != null){ 371 | socket.close(); 372 | } 373 | } catch (IOException e) { 374 | e.printStackTrace(); 375 | } 376 | socket = null; 377 | } 378 | return dataLen; 379 | } 380 | 381 | 382 | // 启动运用通过JVM参数走代理, 注意这种代理是全局的,设置以后全部会自动走代理, 383 | // 如果需要单个请求走代理(在走代理失败的话, 会自动尝试本地直接访问), 请使用proxy_protocol()协议代理函数 384 | // -DproxySet=true 385 | // -Dhttp.proxyHost=proxyIp 386 | // -Dhttp.proxyPort=proxyPort 387 | } 388 | -------------------------------------------------------------------------------- /Java/MimvpProxyJava2.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Java HttpClient 支持 http、https、socks5 3 | * 4 | * HttpClient不是Java JDK默认的类，因此需要添加jar包 5 | * HttpClient jar包下载：http://hc.apache.org/downloads.cgi 6 | * 7 | * 米扑代理示例用的引用jar包如下： 8 | * commons-codec-1.10.jar 9 | * commons-logging-1.2.jar 10 | * httpclient-4.5.5.jar 11 | * httpcore-4.4.9.jar 12 | * 13 | * 米扑代理，已开源此示例和依赖jar包，上传到了Gitub： 14 | * https://github.com/mimvp/mimvp-proxy-demo 15 | * 16 | * 米扑代理示例： 17 | * https://proxy.mimvp.com/demo2.php 18 | * 19 | * 米扑代理购买： 20 | * https://proxy.mimvp.com 21 | * 22 | * mimvp.com 23 | * 2015-11-09 24 | */ 25 | 26 | 27 | package com.mimvp; 28 | 29 | import java.io.IOException; 30 | import java.net.Authenticator; 31 | import java.net.InetAddress; 32 | import java.net.InetSocketAddress; 33 | import java.net.PasswordAuthentication; 34 | import java.net.Proxy; 35 | import java.net.Socket; 36 | import java.net.URL; 37 | import java.net.UnknownHostException; 38 | import java.util.HashMap; 39 | 40 | import javax.net.ssl.SSLContext; 41 | 42 | import org.apache.http.HttpHost; 43 | import org.apache.http.HttpResponse; 44 | import org.apache.http.HttpStatus; 45 | import org.apache.http.auth.AuthScope; 46 | import org.apache.http.auth.UsernamePasswordCredentials; 47 | import org.apache.http.client.CredentialsProvider; 48 | import org.apache.http.client.HttpClient; 49 | import org.apache.http.client.config.RequestConfig; 50 | import org.apache.http.client.methods.HttpGet; 51 | import org.apache.http.client.protocol.HttpClientContext; 52 | import org.apache.http.config.Registry; 53 | import org.apache.http.config.RegistryBuilder; 54 | import org.apache.http.conn.DnsResolver; 55 | import org.apache.http.conn.socket.ConnectionSocketFactory; 56 | import org.apache.http.conn.socket.PlainConnectionSocketFactory; 57 | import org.apache.http.conn.ssl.SSLConnectionSocketFactory; 58 | import org.apache.http.impl.client.BasicCredentialsProvider; 59 | import org.apache.http.impl.client.CloseableHttpClient; 60 | import org.apache.http.impl.client.DefaultHttpClient; 61 | import org.apache.http.impl.client.HttpClientBuilder; 62 | import org.apache.http.impl.client.HttpClients; 63 | import org.apache.http.impl.conn.DefaultProxyRoutePlanner; 64 | import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 65 | import org.apache.http.protocol.HttpContext; 66 | import org.apache.http.ssl.SSLContexts; 67 | import org.apache.http.util.EntityUtils; 68 | 69 | public class MimvpProxyJava2 { 70 | 71 | @SuppressWarnings({ "serial" }) 72 | public static HashMap proxyMap = new HashMap() { 73 | { 74 | put("http", "138.68.161.14:3128"); 75 | put("https", "39.137.37.8:80"); 76 | put("socks4", "222.83.247.90:1080"); 77 | put("socks5", "111.207.55.16:1080"); 78 | } 79 | }; 80 | 81 | 82 | public static String PROXY_USERNAME = "username"; 83 | public static String PROXY_PASSWORD = "password"; 84 | @SuppressWarnings({ "serial" }) 85 | public static HashMap proxyAuthMap = new HashMap() { 86 | { 87 | put("http", "120.24.177.37:4821"); 88 | put("https", "120.24.177.37:4821"); 89 | put("socks5", "120.24.177.37:4825"); 90 | } 91 | }; 92 | 93 | 94 | final static String proxyUrl = "http://proxy.mimvp.com/test_proxy2.php"; // http 95 | final static String proxyUrl2 = "https://proxy.mimvp.com/test_proxy2.php"; // https 96 | 97 | 98 | // 主入口函数 99 | public static void main(String args[]){ 100 | // 通过API实时获取米扑代理 101 | spider_proxy(); 102 | 103 | 104 | // 代理无用户名密码授权 105 | proxy_no_auth("http", MimvpProxyJava2.proxyMap.get("http"), proxyUrl); // http（proxy） 106 | proxy_no_auth("https", MimvpProxyJava2.proxyMap.get("https"), proxyUrl2); // https（proxy） 107 | proxy_no_auth2("http", MimvpProxyJava2.proxyMap.get("http"), proxyUrl); // http（config） 108 | proxy_no_auth3("http", MimvpProxyJava2.proxyMap.get("http"), proxyUrl); // http（routePlanner） 109 | 110 | proxy_no_auth_socks("socks5", MimvpProxyJava2.proxyMap.get("socks5"), proxyUrl); // socks5支持访问http网址 111 | proxy_no_auth_socks("socks5", MimvpProxyJava2.proxyMap.get("socks5"), proxyUrl2); // socks5支持访问https网址 112 | 113 | 114 | // 代理需要用户名密码授权 115 | proxy_auth("http", MimvpProxyJava2.proxyAuthMap.get("http"), proxyUrl); // http 116 | proxy_auth("https", MimvpProxyJava2.proxyAuthMap.get("https"), proxyUrl2); // https 117 | proxy_auth_socks("socks5", MimvpProxyJava2.proxyAuthMap.get("socks5"), proxyUrl); // socks5支持访问http网址 118 | proxy_auth_socks("socks5", MimvpProxyJava2.proxyAuthMap.get("socks5"), proxyUrl2); // socks5支持访问https网址 119 | 120 | } 121 | 122 | 123 | // 通过API实时获取米扑代理 124 | public static void spider_proxy() { 125 | String proxy_url = "https://proxyapi.mimvp.com/api/fetchsecret.php?orderid=868435221231212345&http_type=3"; 126 | 127 | try { 128 | @SuppressWarnings({ "resource", "deprecation" }) 129 | HttpClient client = new DefaultHttpClient(); // 舍弃的用法 130 | CloseableHttpClient client2 = HttpClients.createDefault(); // 推荐的用法 131 | 132 | HttpGet request = new HttpGet(proxy_url); 133 | HttpResponse response = client2.execute(request); 134 | 135 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 136 | String result = EntityUtils.toString(response.getEntity()); 137 | 138 | String[] proxy_list; 139 | proxy_list = result.split("\n"); 140 | for (String proxy : proxy_list) { 141 | System.out.println(proxy); 142 | } 143 | } 144 | client2.close(); 145 | } catch (Exception e) { 146 | System.out.println(e.toString()); 147 | } 148 | } 149 | 150 | 151 | 152 | 153 | // ############################## 代理无授权 ############################## 154 | 155 | // 方法1：无密代理, 支持 http、https（proxy） 156 | public static void proxy_no_auth(String proxyType, String proxyIpPort, String webUrl) { 157 | System.out.println(proxyType + " , " + proxyIpPort + " , " + webUrl); 158 | String proxy_ip = proxyIpPort.split(":")[0]; 159 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 160 | 161 | try { 162 | CloseableHttpClient client = HttpClients.createDefault(); 163 | HttpHost proxy = new HttpHost(proxy_ip, proxy_port); 164 | 165 | HttpGet request = new HttpGet(webUrl); 166 | request.addHeader("Host","proxy.mimvp.com"); 167 | request.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); 168 | 169 | HttpResponse response = client.execute(proxy, request); 170 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 171 | String result = EntityUtils.toString(response.getEntity()); 172 | System.out.println(result); 173 | } 174 | client.close(); 175 | } catch (Exception e) { 176 | System.out.println(e.toString()); 177 | } 178 | } 179 | 180 | 181 | // 方法2：无密代理, 支持 http（config） 182 | public static void proxy_no_auth2(String proxyType, String proxyIpPort, String webUrl) { 183 | System.out.println(proxyType + " , " + proxyIpPort + " , " + webUrl); 184 | String proxy_ip = proxyIpPort.split(":")[0]; 185 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 186 | 187 | try { 188 | HttpClientBuilder builder = HttpClientBuilder.create(); 189 | CloseableHttpClient client = builder.build(); 190 | 191 | URL url = new URL(webUrl); 192 | HttpHost target = new HttpHost(url.getHost(), url.getDefaultPort(), url.getProtocol()); 193 | if(proxyType.equals("https")) { 194 | target = new HttpHost(url.getHost(), 443, "https"); 195 | } 196 | HttpHost proxy = new HttpHost(proxy_ip, proxy_port); 197 | 198 | RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); 199 | HttpGet request = new HttpGet(url.getPath()); 200 | request.setConfig(config); 201 | request.addHeader("Host","proxy.mimvp.com"); 202 | request.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); 203 | 204 | HttpResponse response = client.execute(target, request); 205 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 206 | String result = EntityUtils.toString(response.getEntity()); 207 | System.out.println(result); 208 | } 209 | client.close(); 210 | } catch (Exception e) { 211 | System.out.println(e.toString()); 212 | } 213 | } 214 | 215 | 216 | // 方法3：无密代理, 支持 http（routePlanner） 217 | public static void proxy_no_auth3(String proxyType, String proxyIpPort, String webUrl) { 218 | System.out.println(proxyType + " , " + proxyIpPort + " , " + webUrl); 219 | String proxy_ip = proxyIpPort.split(":")[0]; 220 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 221 | 222 | try { 223 | HttpHost proxy = new HttpHost(proxy_ip, proxy_port); 224 | DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy); 225 | 226 | CloseableHttpClient client = HttpClients.custom().setRoutePlanner(routePlanner).build(); 227 | HttpGet request = new HttpGet(webUrl); 228 | request.addHeader("Host","proxy.mimvp.com"); 229 | request.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); 230 | 231 | HttpResponse response = client.execute(request); 232 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 233 | String result = EntityUtils.toString(response.getEntity()); 234 | System.out.println(result); 235 | } 236 | client.close(); 237 | } catch (Exception e) { 238 | System.out.println(e.toString()); 239 | } 240 | } 241 | 242 | 243 | 244 | 245 | // ############################## 代理用户名密码授权 ############################## 246 | 247 | // HttpClient 支持socks5代理的自定义类 248 | static class MyConnectionSocketFactory extends PlainConnectionSocketFactory { 249 | @Override 250 | public Socket createSocket(final HttpContext context) throws IOException { 251 | InetSocketAddress socksaddr = (InetSocketAddress) context.getAttribute("socks.address"); 252 | Proxy proxy = new Proxy(Proxy.Type.SOCKS, socksaddr); 253 | return new Socket(proxy); 254 | } 255 | 256 | @Override 257 | public Socket connectSocket(int connectTimeout, Socket socket, HttpHost host, InetSocketAddress remoteAddress, 258 | InetSocketAddress localAddress, HttpContext context) throws IOException { 259 | InetSocketAddress unresolvedRemote = InetSocketAddress.createUnresolved(host.getHostName(), remoteAddress.getPort()); 260 | return super.connectSocket(connectTimeout, socket, host, unresolvedRemote, localAddress, context); 261 | } 262 | } 263 | 264 | static class MySSLConnectionSocketFactory extends SSLConnectionSocketFactory { 265 | public MySSLConnectionSocketFactory(final SSLContext sslContext) { 266 | // super(sslContext, ALLOW_ALL_HOSTNAME_VERIFIER); 267 | super(sslContext); 268 | } 269 | 270 | @Override 271 | public Socket createSocket(final HttpContext context) throws IOException { 272 | InetSocketAddress socksaddr = (InetSocketAddress) context.getAttribute("socks.address"); 273 | Proxy proxy = new Proxy(Proxy.Type.SOCKS, socksaddr); 274 | return new Socket(proxy); 275 | } 276 | 277 | @Override 278 | public Socket connectSocket(int connectTimeout, Socket socket, HttpHost host, InetSocketAddress remoteAddress, 279 | InetSocketAddress localAddress, HttpContext context) throws IOException { 280 | InetSocketAddress unresolvedRemote = InetSocketAddress.createUnresolved(host.getHostName(), remoteAddress.getPort()); 281 | return super.connectSocket(connectTimeout, socket, host, unresolvedRemote, localAddress, context); 282 | } 283 | } 284 | 285 | static class FakeDnsResolver implements DnsResolver { 286 | @Override 287 | public InetAddress[] resolve(String host) throws UnknownHostException { 288 | // Return some fake DNS record for every request, we won't be using it 289 | return new InetAddress[] { InetAddress.getByAddress(new byte[] { 1, 1, 1, 1 }) }; 290 | } 291 | } 292 | 293 | // 无密代理, 支持 socks5 294 | public static void proxy_no_auth_socks(String proxyType, String proxyIpPort, String webUrl) { 295 | System.out.println(proxyType + " , " + proxyIpPort + " , " + webUrl); 296 | String proxy_ip = proxyIpPort.split(":")[0]; 297 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 298 | 299 | try { 300 | Registry reg = RegistryBuilder. create() 301 | .register("http", new MyConnectionSocketFactory()) 302 | .register("https", new MySSLConnectionSocketFactory(SSLContexts.createSystemDefault())) 303 | .build(); 304 | PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(reg, new FakeDnsResolver()); 305 | CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build(); 306 | 307 | InetSocketAddress addr = new InetSocketAddress(proxy_ip, proxy_port); 308 | HttpClientContext context = HttpClientContext.create(); 309 | context.setAttribute("socks.address", addr); 310 | 311 | HttpGet request = new HttpGet(webUrl); 312 | request.addHeader("Host","proxy.mimvp.com"); 313 | request.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); 314 | 315 | HttpResponse response = client.execute(request, context); 316 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 317 | String result = EntityUtils.toString(response.getEntity()); 318 | System.out.println(result); 319 | } 320 | } catch (Exception e) { 321 | System.out.println(e.toString()); 322 | } 323 | } 324 | 325 | 326 | 327 | // 有密代理，需要用户名密码授权，请先取消授权的注释（代码里有注释说明） 328 | public static void proxy_auth(String proxyType, String proxyIpPort, String webUrl) { 329 | System.out.println(proxyType + " , " + proxyIpPort + " , " + webUrl); 330 | String proxy_ip = proxyIpPort.split(":")[0]; 331 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 332 | 333 | CredentialsProvider provider = new BasicCredentialsProvider(); 334 | provider.setCredentials( new AuthScope(proxy_ip, proxy_port), 335 | new UsernamePasswordCredentials(MimvpProxyJava2.PROXY_USERNAME, MimvpProxyJava2.PROXY_PASSWORD)); 336 | 337 | CloseableHttpClient client = HttpClients.custom().setDefaultCredentialsProvider(provider).build(); 338 | try { 339 | URL url = new URL(webUrl); 340 | HttpHost target = new HttpHost(url.getHost(),url.getDefaultPort(),url.getProtocol()); 341 | HttpHost proxy = new HttpHost(proxy_ip, proxy_port); 342 | 343 | RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); 344 | HttpGet request = new HttpGet(url.getPath()); 345 | request.setConfig(config); 346 | request.addHeader("Host","proxy.mimvp.com"); 347 | request.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); 348 | 349 | HttpResponse response = client.execute(target, request); 350 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 351 | String result = EntityUtils.toString(response.getEntity()); 352 | System.out.println(result); 353 | } 354 | } catch (Exception e) { 355 | e.printStackTrace(); 356 | } 357 | } 358 | 359 | 360 | // socks5代理的用户名密码授权 361 | static class MyAuthenticator extends Authenticator { 362 | private String username = ""; 363 | private String password = ""; 364 | public MyAuthenticator(String username, String password) { 365 | this.username = username; 366 | this.password = password; 367 | } 368 | protected PasswordAuthentication getPasswordAuthentication() { 369 | return new PasswordAuthentication(this.username, this.password.toCharArray()); 370 | } 371 | } 372 | 373 | // 有密代理, 支持 socks5 374 | public static void proxy_auth_socks(String proxyType, String proxyIpPort, String webUrl) { 375 | System.out.println(proxyType + " , " + proxyIpPort + " , " + webUrl); 376 | String proxy_ip = proxyIpPort.split(":")[0]; 377 | int proxy_port = Integer.parseInt(proxyIpPort.split(":")[1]); 378 | 379 | try { 380 | Registry reg = RegistryBuilder. create() 381 | .register("http", new MyConnectionSocketFactory()) 382 | .register("https", new MySSLConnectionSocketFactory(SSLContexts.createSystemDefault())) 383 | .build(); 384 | PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(reg, new FakeDnsResolver()); 385 | CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build(); 386 | 387 | // auth 代理需要用户名密码授权时开启，取消此注释，米扑代理验证通过 388 | Authenticator.setDefault(new MyAuthenticator(MimvpProxyJava2.PROXY_USERNAME, MimvpProxyJava2.PROXY_PASSWORD)); 389 | 390 | InetSocketAddress addr = new InetSocketAddress(proxy_ip, proxy_port); 391 | HttpClientContext context = HttpClientContext.create(); 392 | context.setAttribute("socks.address", addr); 393 | 394 | HttpGet request = new HttpGet(webUrl); 395 | request.addHeader("Host","proxy.mimvp.com"); 396 | request.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); 397 | 398 | HttpResponse response = client.execute(request, context); 399 | if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { 400 | String result = EntityUtils.toString(response.getEntity()); 401 | System.out.println(result); 402 | } 403 | } catch (Exception e) { 404 | System.out.println(e.toString()); 405 | } 406 | } 407 | } 408 | -------------------------------------------------------------------------------- /Java/lib/commons-codec-1.10.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Java/lib/commons-codec-1.10.jar -------------------------------------------------------------------------------- /Java/lib/commons-logging-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Java/lib/commons-logging-1.2.jar -------------------------------------------------------------------------------- /Java/lib/httpclient-4.5.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Java/lib/httpclient-4.5.5.jar -------------------------------------------------------------------------------- /Java/lib/httpcomponents-client-4.5.5-bin.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Java/lib/httpcomponents-client-4.5.5-bin.tar.gz -------------------------------------------------------------------------------- /Java/lib/httpcomponents-client-4.5.5-src.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Java/lib/httpcomponents-client-4.5.5-src.tar.gz -------------------------------------------------------------------------------- /Java/lib/httpcore-4.4.9.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Java/lib/httpcore-4.4.9.jar -------------------------------------------------------------------------------- /NodeJS/mimvp-proxy-nodejs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * NodeJS 支持 http、https、socks4、socks5 3 | * 4 | * 米扑代理示例： 5 | * http://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * http://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2017.5.25 12 | * 13 | * 米扑博客示例详解 : 14 | * http://blog.mimvp.com/2017/10/node-js-she-zhi-dai-li-de-liang-zhong-fang-shi-superagent-proxy-he-https-proxy-agent/ 15 | * 16 | * proxy-agent : https://www.npmjs.com/package/proxy-agent 17 | * superagent-proxy : https://www.npmjs.com/package/superagent-proxy 18 | * http-proxy-agent : https://www.npmjs.com/package/http-proxy-agent 19 | * https-proxy-agent : https://www.npmjs.com/package/https-proxy-agent 20 | * socks-proxy-agent : https://www.npmjs.com/package/socks-proxy-agent 21 | */ 22 | 23 | var proxy_http = "http://35.154.138.213:80"; 24 | var proxy_https = "http://103.247.154.105:8080"; 25 | var proxy_socks4 = "socks4://202.159.8.243:1080"; 26 | var proxy_socks5 = "socks5://94.177.216.47:2016"; 27 | 28 | var mimvp_url = "http://proxy.mimvp.com/exist.php"; 29 | var mimvp_url2 = "https://proxy.mimvp.com/exist.php"; 30 | var mimvp_url3 = "https://apps.bdimg.com/libs/jquery-i18n/1.1.1/jquery.i18n.min.js"; 31 | 32 | 33 | 34 | //http for http 35 | function check_http(proxy_uri, mimvp_url) { 36 | var http = require('http'); 37 | var url = require('url'); 38 | 39 | console.log("proxy_uri : " + proxy_uri); 40 | 41 | var proxy_ip = proxy_uri.toString().split("://")[1].split(":")[0]; // IP 42 | var proxy_port = proxy_uri.toString().split("://")[1].split(":")[1]; // Port 43 | 44 | const options = { 45 | host : proxy_ip, 46 | port : proxy_port, 47 | path : mimvp_url, 48 | method : 'GET', 49 | headers : { 50 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", 51 | "Host": "apps.bdimg.com", 52 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 53 | } 54 | }; 55 | 56 | http 57 | .request(options, function(res) { 58 | console.log("got response code: " + res.statusCode); 59 | if (res.statusCode == 200) { 60 | res.setEncoding("utf-8"); 61 | var resData = ""; 62 | res.on("data", function(chunk) { 63 | resData += chunk; 64 | }).on("end", function() { 65 | console.log("got response text: " + resData); 66 | }); 67 | } else { 68 | console.log("res.statusCode err " + res.statusCode); // 请求失败 69 | } 70 | }) 71 | .on("error", function(err) { 72 | console.log(err); 73 | }) 74 | .end(); 75 | }; 76 | 77 | 78 | // request http, https 79 | function check_request(proxy_uri, mimvp_url) { 80 | var request = require('request'); 81 | 82 | console.log("proxy_uri : " + proxy_uri); 83 | 84 | const proxiedRequest = request.defaults({'proxy': proxy_uri}); 85 | const options = { 86 | url : mimvp_url, 87 | headers : { 88 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", 89 | "Host": "apps.bdimg.com", 90 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 91 | } 92 | }; 93 | 94 | proxiedRequest 95 | .get(options, function (err, res, body) { 96 | console.log("got response: " + res.statusCode); 97 | console.log("got body: " + body); 98 | }) 99 | .on("error", function (err) { 100 | console.log(err); 101 | }); 102 | }; 103 | 104 | 105 | // superagent-proxy http, https 106 | function check_superagent(proxy_uri, mimvp_url) { 107 | var superagent = require('superagent'); 108 | require('superagent-proxy')(superagent); 109 | 110 | console.log("proxy_uri : " + proxy_uri); 111 | 112 | superagent 113 | .get(mimvp_url) 114 | .proxy(proxy_uri) 115 | .buffer(true) 116 | .set('User-Agent','Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36') 117 | .set('Host','apps.bdimg.com') 118 | .set('Accept','text/html,application/xhtml+xml,application/xml,application/x-javascript;q=0.9,image/webp,image/apng,*/*;q=0.8') 119 | .set('Accept-Encoding','gzip, deflate, br') 120 | .end(function onResponse(err, res) { 121 | if (err) { 122 | console.log(err); 123 | } else { 124 | console.log(res.status, res.headers); 125 | console.log(res.text); 126 | } 127 | }); 128 | }; 129 | 130 | 131 | // https-proxy-agent https 132 | function check_httpsproxyagent(proxy_uri, mimvp_url) { 133 | var url = require('url'); 134 | var https = require('https'); 135 | var HttpsProxyAgent = require('https-proxy-agent'); 136 | 137 | console.log('using proxy server %j', proxy_uri); // 'https://79.137.80.210:3128' 138 | 139 | var agent = new HttpsProxyAgent(proxy_uri); 140 | var endpoint = process.argv[2] || mimvp_url; 141 | var options = url.parse(endpoint); 142 | options.agent = agent; 143 | options.port = 443; 144 | options.secureProxy = true; 145 | 146 | headers = { 147 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", 148 | "Host": "apps.bdimg.com", 149 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 150 | }; 151 | options.headers = headers; 152 | console.log('options %j', options); 153 | 154 | https.get(options, function (res) { 155 | console.log('"response" event!', res.headers); 156 | res.pipe(process.stdout); 157 | }); 158 | } 159 | 160 | 161 | //socks-proxy-agent socks4, socks5 162 | function check_socksproxyagent(proxy_uri, mimvp_url) { 163 | var url = require('url'); 164 | var http = require('http'); 165 | var SocksProxyAgent = require('socks-proxy-agent'); 166 | 167 | console.log('using proxy server %j', proxy_uri); // 'socks://94.177.216.47:2016' 168 | 169 | var agent = new SocksProxyAgent(proxy_uri); 170 | var endpoint = process.argv[2] || mimvp_url; 171 | var options = url.parse(endpoint); 172 | options.agent = agent; 173 | options.port = 443; 174 | options.secureProxy = true; 175 | 176 | headers = { 177 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", 178 | "Host": "apps.bdimg.com", 179 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 180 | "Accept-Encoding": "gzip, deflate, br", 181 | }; 182 | options.headers = headers; 183 | console.log('options %j', options); 184 | 185 | http.get(options, function (res) { 186 | console.log('"response" event!', res.headers); 187 | res.pipe(process.stdout); 188 | }); 189 | } 190 | 191 | // http 192 | check_http(proxy_http, mimvp_url); // http 193 | 194 | // request 195 | check_request(proxy_http, mimvp_url); // http 196 | check_request(proxy_https, mimvp_url3); // https 197 | 198 | // superagent 199 | check_superagent(proxy_http, mimvp_url); // http 200 | check_superagent(proxy_https, mimvp_url3); // https 201 | 202 | // https_proxy_agent 203 | check_httpsproxyagent(proxy_https, mimvp_url3); // https 204 | 205 | // socks_proxy_agent 206 | check_socksproxyagent(proxy_socks4, mimvp_url); // socks4 207 | check_socksproxyagent(proxy_socks5, mimvp_url); // socks5 208 | 209 | 210 | // 执行命令： 211 | // cd /usr/local/node/lib/node_modules/npm/ 212 | // sudo cp ~/Documents/workspace/MimvpProxyDemo/NodeJS/mimvp-proxy-nodejs.js . 213 | // sudo node mimvp-proxy-nodejs.js | grep '' 214 | 215 | -------------------------------------------------------------------------------- /PHP/mimvp-proxy-php.php: -------------------------------------------------------------------------------- 1 | $proxy"; 50 | } 51 | } 52 | 53 | 54 | 55 | # 实时爬取米扑代理API接口 (stream) 56 | function spider_proxy_by_stream() { 57 | $proxy_url = "https://proxyapi.mimvp.com/api/fetchsecret.php?orderid=868435225671251234&http_type=3"; 58 | 59 | $header_array = array( "Accept-Language:zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4", 60 | "Referer:http://proxy.mimvp.com/fetch.php", 61 | "Host:proxy.mimvp.com", 62 | "User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"); 63 | 64 | $options = array("http" => array( 65 | "header" => $header_array, 66 | "method" => "GET") 67 | ); 68 | 69 | $context = stream_context_create($options); 70 | $result = file_get_contents($proxy_url, false, $context); 71 | var_dump($result); 72 | 73 | $proxy_list = explode("\n", $result); 74 | foreach ($proxy_list as $proxy) { 75 | echo "
$proxy"; 76 | } 77 | } 78 | 79 | 80 | 81 | ######################### php curl 支持 http、https、socks4、socks5（代理无密码）######################### 82 | 83 | $proxy_http = "http://118.89.57.38:8080"; 84 | $proxy_https = "https://39.134.108.92:8080"; 85 | $proxy_socks4 = "socks4://59.37.163.176:1080"; 86 | $proxy_socks5 = "socks5://183.156.71.161:1080"; 87 | 88 | function proxy_curl($proxy_uri, $mimvp_url) { 89 | $proxy_type = explode('://', $proxy_uri)[0]; // http, https, socks4, socks5 90 | $proxy_ip_port = explode('://', $proxy_uri)[1]; // ip:port 91 | echo "proxy_uri: $proxy_uri ; proxy_type: $proxy_type , proxy_ip_port: $proxy_ip_port
"; 92 | 93 | $ch = curl_init (); 94 | curl_setopt ( $ch, CURLOPT_URL, $mimvp_url); 95 | curl_setopt ( $ch, CURLOPT_HTTPPROXYTUNNEL, false ); 96 | curl_setopt ( $ch, CURLOPT_PROXY, $proxy_ip_port ); 97 | 98 | if ($proxy_type == "http") { 99 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP ); // http 100 | } 101 | elseif ($proxy_type == "https") { 102 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYHOST, 2 ); 103 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYPEER, false ); // https 104 | } 105 | elseif ($proxy_type == "socks4") { 106 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4 ); // socks4 107 | } 108 | elseif ($proxy_type == "socks5") { 109 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5 ); // socks5 110 | } 111 | 112 | curl_setopt ( $ch, CURLOPT_TIMEOUT, 60 ); 113 | curl_setopt ( $ch, CURLOPT_CONNECTTIMEOUT, 60 ); 114 | curl_setopt ( $ch, CURLOPT_HEADER, false ); 115 | curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, true ); // 返回网页内容 116 | 117 | $result = curl_exec ( $ch ); 118 | print_r($result); // 打印网页正文 119 | 120 | $curl_info = curl_getinfo($ch); // 打印网页信息 121 | var_dump($curl_info); 122 | echo $curl_info['size_download'] . "

"; 123 | 124 | curl_close ( $ch ); 125 | } 126 | 127 | 128 | 129 | 130 | ######################### php curl 支持 http、https、socks5（代理有密码）######################### 131 | 132 | $proxy_auth_http = "http://username:password@125.121.168.231:5140"; 133 | $proxy_auth_https = "https://username:password@125.121.168.231:5140"; 134 | $proxy_auth_socks5 = "socks5://username:password@125.121.168.231:5141"; 135 | 136 | function proxy_auth_curl($proxy_uri, $mimvp_url) { 137 | $proxy_type = explode('://', $proxy_uri)[0]; // http, https, socks4, socks5 138 | $proxy_ip_port = explode('://', $proxy_uri)[1]; // ip:port 139 | echo "proxy_uri: $proxy_uri ; proxy_type: $proxy_type , proxy_ip_port: $proxy_ip_port
"; 140 | 141 | $ch = curl_init (); 142 | curl_setopt ( $ch, CURLOPT_URL, $mimvp_url); 143 | curl_setopt ( $ch, CURLOPT_HTTPPROXYTUNNEL, false ); 144 | curl_setopt ( $ch, CURLOPT_PROXY, $proxy_ip_port ); 145 | 146 | if ($proxy_type == "http") { 147 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP ); // http 148 | } 149 | elseif ($proxy_type == "https") { 150 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYHOST, 2 ); 151 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYPEER, false ); // https 152 | } 153 | elseif ($proxy_type == "socks5") { 154 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5 ); // socks5 155 | } 156 | 157 | curl_setopt ( $ch, CURLOPT_TIMEOUT, 60 ); 158 | curl_setopt ( $ch, CURLOPT_CONNECTTIMEOUT, 60 ); 159 | curl_setopt ( $ch, CURLOPT_HEADER, false ); 160 | curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, true ); // 返回网页内容 161 | 162 | $result = curl_exec ( $ch ); 163 | print_r($result); // 打印网页正文 164 | 165 | $curl_info = curl_getinfo($ch); // 打印网页信息 166 | var_dump($curl_info); 167 | echo $curl_info['size_download'] . "

"; 168 | 169 | curl_close ( $ch ); 170 | } 171 | 172 | 173 | 174 | 175 | ######################### php curl 支持 http、https、socks5（代理有密码）######################### 176 | 177 | $proxy_noauth_http = "http://125.121.168.231:5140"; 178 | $proxy_noauth_https = "https://125.121.168.231:5140"; 179 | $proxy_noauth_socks5 = "socks5://125.121.168.231:5141"; 180 | 181 | $PROXY_USERNAME = 'username'; 182 | $PROXY_PASSEORD = 'password'; 183 | 184 | function proxy_noauth_curl($proxy_uri, $mimvp_url) { 185 | global $PROXY_USERNAME; 186 | global $PROXY_PASSEORD; 187 | 188 | $proxy_type = explode('://', $proxy_uri)[0]; // http, https, socks4, socks5 189 | $proxy_ip_port = explode('://', $proxy_uri)[1]; // ip:port 190 | echo "proxy_uri: $proxy_uri ; proxy_type: $proxy_type , proxy_ip_port: $proxy_ip_port
"; 191 | 192 | $ch = curl_init (); 193 | curl_setopt ( $ch, CURLOPT_URL, $mimvp_url); 194 | curl_setopt ( $ch, CURLOPT_HTTPPROXYTUNNEL, false ); 195 | curl_setopt ( $ch, CURLOPT_PROXY, $proxy_ip_port ); 196 | 197 | # 设置代理授权 198 | curl_setopt($ch, CURLOPT_PROXYAUTH, CURLAUTH_BASIC); 199 | curl_setopt($ch, CURLOPT_PROXYUSERPWD, "{$PROXY_USERNAME}:{$PROXY_PASSEORD}"); 200 | 201 | if ($proxy_type == "http") { 202 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP ); // http 203 | } 204 | elseif ($proxy_type == "https") { 205 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYHOST, 2 ); 206 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYPEER, false ); // https 207 | } 208 | elseif ($proxy_type == "socks5") { 209 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5 ); // socks5 210 | } 211 | 212 | curl_setopt ( $ch, CURLOPT_TIMEOUT, 60 ); 213 | curl_setopt ( $ch, CURLOPT_CONNECTTIMEOUT, 60 ); 214 | curl_setopt ( $ch, CURLOPT_HEADER, false ); 215 | curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, true ); // 返回网页内容 216 | 217 | $result = curl_exec ( $ch ); 218 | print_r($result); // 打印网页正文 219 | 220 | $curl_info = curl_getinfo($ch); // 打印网页信息 221 | var_dump($curl_info); 222 | echo $curl_info['size_download'] . "

"; 223 | 224 | curl_close ( $ch ); 225 | } 226 | 227 | 228 | 229 | 230 | ######################### php stream 支持 tcp, 含http和https代理 ######################### 231 | 232 | $proxy_tcp = "tcp://118.89.57.38:8080"; // 支持http 233 | $proxy_tcp = "tcp://39.134.108.92:8080"; // 支持https 234 | 235 | function proxy_stream($proxy_uri, $mimvp_url) { 236 | $proxy_type = explode('://', $proxy_uri)[0]; // http, https 237 | $proxy_ip_port = explode('://', $proxy_uri)[1]; // ip:port 238 | echo "proxy_uri: $proxy_uri ; proxy_type: $proxy_type , proxy_ip_port: $proxy_ip_port
"; 239 | 240 | $header_array = array( "Accept-Language:zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4", 241 | "Referer:http://proxy.mimvp.com/fetch.php", 242 | "Host:proxy.mimvp.com", 243 | "User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"); 244 | 245 | $options = array("http" => array( 246 | "proxy" => $proxy_uri, 247 | "header" => $header_array, 248 | "method" => "GET") 249 | ); 250 | 251 | $context = stream_context_create($options); 252 | $result = file_get_contents($mimvp_url, false, $context); 253 | 254 | print_r($result); 255 | } 256 | 257 | 258 | // 通过API获取米扑代理 259 | spider_proxy_by_curl(); 260 | spider_proxy_by_stream(); 261 | 262 | 263 | // curl proxy no proxy 264 | proxy_curl($proxy_http, $mimvp_url); // http 265 | proxy_curl($proxy_https, $mimvp_url); // https 266 | proxy_curl($proxy_socks4, $mimvp_url); // socks4 267 | proxy_curl($proxy_socks5, $mimvp_url); // socks5 268 | 269 | 270 | // 授权方法1： curl proxy proxy 271 | proxy_auth_curl($proxy_auth_http, $mimvp_url); // http 272 | proxy_auth_curl($proxy_auth_https, $mimvp_url); // https 273 | proxy_auth_curl($proxy_auth_socks5, $mimvp_url); // socks5 访问http 274 | proxy_auth_curl($proxy_auth_socks5, $mimvp_url2); // socks5 访问https 275 | 276 | 277 | // 授权方法2： curl proxy proxy 278 | proxy_noauth_curl($proxy_noauth_http, $mimvp_url); // http 279 | proxy_noauth_curl($proxy_noauth_https, $mimvp_url); // https 280 | proxy_noauth_curl($proxy_noauth_socks5, $mimvp_url); // socks5 访问http 281 | proxy_noauth_curl($proxy_noauth_socks5, $mimvp_url2); // socks5 访问https 282 | 283 | 284 | // stream 285 | proxy_stream($proxy_tcp, $mimvp_url); // tcp http 286 | proxy_stream($proxy_tcp, $mimvp_url2); // tcp https 287 | 288 | ?> -------------------------------------------------------------------------------- /Perl/mimvp-proxy-perl.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | # 3 | # Perl 支持 http、https 4 | # 5 | # 米扑代理示例： 6 | # http://proxy.mimvp.com/demo2.php 7 | # 8 | # 米扑代理购买： 9 | # http://proxy.mimvp.com 10 | # 11 | # mimvp.com 12 | # 2017-03-28 13 | 14 | 15 | use CGI; 16 | use strict; 17 | use LWP::UserAgent; 18 | 19 | 20 | our %proxy_http = ("http", "http://138.68.165.154:3128"); 21 | our %proxy_https = ("https", "https://113.106.94.213:80"); 22 | our %proxy_connect = ("https", "connect://173.233.55.118:443"); 23 | 24 | our $mimvp_url = "http://proxy.mimvp.com/exist.php"; 25 | our $mimvp_url2 = "https://proxy.mimvp.com/exist.php"; 26 | 27 | 28 | 29 | ## http 30 | sub test_http { 31 | my ($url, %proxy) = @_; 32 | 33 | print "proxy : $proxy{'http'}\n"; 34 | print "https : $proxy{'https'}\n"; 35 | print "url : $url\n"; 36 | 37 | my $browser = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 }); 38 | $browser->env_proxy(); 39 | 40 | # # 设置的代理格式 41 | # $browser->proxy('http', 'http://138.68.165.154:3128'); 42 | # $browser->proxy(['http','ftp'], 'http://138.68.165.154:3128'); 43 | $browser->proxy(%proxy); 44 | $browser->timeout(30); 45 | $browser->agent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'); 46 | 47 | my $response = $browser->get($url); # 爬取的网址 48 | my $is_success = $response->is_success(); # 1 49 | my $content_type = $response->content_type(); # text/html 50 | my $content = $response->content(); # 网页正文 51 | my $content_length = length($content); # 网页正文长度 52 | 53 | print "$is_success\n"; 54 | print "$content_type\n"; 55 | print "$content_length\n"; 56 | print "$content\n"; 57 | } 58 | 59 | 60 | 61 | ## https (NO Success) 62 | ## error info : LWP::Protocol::https::Socket: SSL connect attempt failed because of handshake problems SSL wants a read first at /System/Library/Perl/Extras/5.18/LWP/Protocol/http.pm line 51. 63 | sub test_https { 64 | my ($url, %proxy) = @_; 65 | 66 | print "proxy : $proxy{'http'}\n"; 67 | print "https : $proxy{'https'}\n"; 68 | print "url : $url\n"; 69 | 70 | BEGIN { 71 | $ENV{HTTPS_PROXY} = 'https://173.233.55.118:443'; 72 | # $ENV{HTTPS_PROXY_USERNAME} = ; 73 | # $ENV{HTTPS_PROXY_PASSWORD} = ; 74 | $ENV{HTTPS_DEBUG} = 1; 75 | $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = 0; 76 | } 77 | 78 | my $browser = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 }); 79 | $browser->env_proxy(); 80 | 81 | # # 设置的代理格式 82 | # $browser->proxy(%proxy); # NO USE 83 | $browser->timeout(30); 84 | $browser->agent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'); 85 | 86 | my $req = new HTTP::Request('GET', $url); # 爬取的网址 87 | my $response = $browser->request($req); 88 | 89 | my $is_success = $response->is_success(); # 1 90 | my $content_type = $response->content_type(); # text/html 91 | my $content = $response->content(); # 网页正文 92 | my $content_length = length($content); # 网页正文长度 93 | 94 | print "$is_success\n"; 95 | print "$content_type\n"; 96 | print "$content_length\n"; 97 | print "$content\n"; 98 | } 99 | 100 | 101 | 102 | ## https (connect) 103 | ## 1. download LWP-Protocol-connect (wget http://search.cpan.org/CPAN/authors/id/B/BE/BENNING/LWP-Protocol-connect-6.09.tar.gz) 104 | ## 2. tar zxvf LWP-Protocol-connect-6.09.tar.gz 105 | ## cd LWP-Protocol-connect-6.09 106 | ## perl Makefile.PL 107 | ## make 108 | ## sudo make install 109 | sub test_connect { 110 | my ($url, %proxy) = @_; 111 | 112 | print "proxy : $proxy{'http'}\n"; 113 | print "https : $proxy{'https'}\n"; 114 | print "url : $url\n"; 115 | 116 | my $browser = LWP::UserAgent->new(); 117 | $browser->env_proxy(); 118 | 119 | # # 设置的代理格式 120 | # $browser->proxy('https', 'connect://173.233.55.118:443'); 121 | $browser->proxy(%proxy); 122 | $browser->timeout(30); 123 | $browser->agent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'); 124 | 125 | # my $req = new HTTP::Request('GET', $url); 126 | # my $response = $browser->request($req); 127 | my $response = $browser->get($url); # 爬取的网址 128 | my $is_success = $response->is_success(); # 1 129 | my $content_type = $response->content_type(); # text/html 130 | my $content = $response->content(); # 网页正文 131 | my $content_length = length($content); # 网页正文长度 132 | 133 | print "$is_success\n"; 134 | print "$content_type\n"; 135 | print "$content_length\n"; 136 | print "$content\n"; 137 | } 138 | 139 | 140 | test_http($mimvp_url, %proxy_http); # http 141 | test_https($mimvp_url2, %proxy_https); # https (NO Success) 142 | test_connect($mimvp_url2, %proxy_connect); # https (connect) 143 | 144 | 145 | 146 | ## 执行命令 147 | ## perl -d mimvp-proxy-perl.pl | grep '' 148 | 149 | 150 | 151 | 152 | 153 | # ## 普通变量 154 | # $num = 2066; # 整型 155 | # $name = "mimvp.com"; # 字符串 156 | # $salary = 1000000.52; # 浮点数 157 | # 158 | # print "num = $num\n"; # 2066 159 | # print "name = $name\n"; # mimvp.com 160 | # print "salary = $salary\n"; # 1000000.52 161 | # 162 | # 163 | # ## 数组变量 164 | # @num_array = (20, 30, 40); 165 | # @name_array = ("proxy.mimvp.com", "domain.mimvp.com", "money.mimvp.com"); 166 | # 167 | # print "\$num_array : @num_array\n"; # 20 30 40 168 | # print "\$num_array[0] = $num_array[0]\n"; # 20 169 | # print "\$num_array[2] = $num_array[2]\n"; # 40 170 | # print "\$num_array[-1] = $num_array[-1]\n"; # 40 171 | # print "\$name_array[1] = $name_array[1]\n"; # domain.mimvp.com 172 | # print "\$name_array[-1] = $name_array[-1]\n"; # money.mimvp.com 173 | # 174 | # 175 | # ## 哈希变量 176 | # %data_dict = ('proxy', 20, 'domain', 30, 'money', 40); 177 | # 178 | # print "\$data_dict{'proxy'} = $data_dict{'proxy'}\n"; # 20 179 | # print "\$data_dict{'domain'} = $data_dict{'domain'}\n"; # 30 180 | # print "\$data_dict{'money'} = $data_dict{'money'}\n"; # 40 181 | # 182 | # 183 | # ## 184 | # @name_array = ('taobao', 'tencent', 'baidu', 'mimvp'); 185 | # 186 | # @copy_array = @name_array; # 复制数组 187 | # $array_size = @copy_array; # 返回数组元素个数 188 | # 189 | # print "copy_array : @copy_array\n"; # taobao tencent baidu mimvp 190 | # print "array_size : $array_size\n"; # 4 191 | 192 | 193 | 194 | 195 | 196 | 197 | # my $mimvp = "mimvp.com"; 198 | # print "$mimvp\n"; # mimvp.com 199 | # 200 | # myFunction(); 201 | # print "$mimvp\n"; # mimvp.com 202 | # 203 | # sub myFunction { 204 | # print "$mimvp\n"; # mimvp.com 205 | # 206 | # my $mimvp = "mimvp.com in myFunction"; 207 | # print "$mimvp\n"; # mimvp.com in myFunction 208 | # mySub(); 209 | # } 210 | # 211 | # sub mySub { 212 | # print "$mimvp\n"; # mimvp.com 213 | # 214 | # my $mimvp = "mimvp.com in mySub"; 215 | # print "$mimvp\n"; # mimvp.com in mySub 216 | # } 217 | 218 | 219 | -------------------------------------------------------------------------------- /PhantomJS/mimvp-proxy-phantomjs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * PhantomJS 支持 http、socks5 3 | * 4 | * 米扑代理示例： 5 | * http://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * http://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2017.5.22 12 | */ 13 | 14 | //http代理格式 ： phantomjs --proxy-type=http --proxy=IP:Port 15 | //socks代理格式 ： phantomjs --proxy-type=socks5 --proxy=IP:Port 16 | 17 | 18 | /** 19 | $ phantomjs -h 20 | --ignore-ssl-errors= Ignores SSL errors (expired/self-signed certificate errors): 'true' or 'false' (default) 21 | --proxy= Sets the proxy server, e.g. '--proxy=http://proxy.company.com:8080' 22 | --proxy-auth= Provides authentication information for the proxy, e.g. ''-proxy-auth=username:password' 23 | --proxy-type= Specifies the proxy type, 'http' (default), 'none' (disable completely), or 'socks5' 24 | --script-encoding= Sets the encoding used for the starting script, default is 'utf8' 25 | --web-security= Enables web security, 'true' (default) or 'false' 26 | --ssl-protocol= Selects a specific SSL protocol version to offer. 27 | --ssl-ciphers= Sets supported TLS/SSL ciphers. 28 | --ssl-certificates-path= Sets the location for custom CA certificates 29 | --ssl-client-certificate-file= Sets the location of a client certificate 30 | --ssl-client-key-file= Sets the location of a clients' private key 31 | --ssl-client-key-passphrase= Sets the passphrase for the clients' private key 32 | */ 33 | 34 | 35 | var mimvp_url = "http://proxy.mimvp.com/exist.php"; 36 | var mimvp_url2 = "https://proxy.mimvp.com/exist.php"; 37 | var mimvp_url3 = "https://apps.bdimg.com/libs/jquery-i18n/1.1.1/jquery.i18n.min.js"; 38 | 39 | var page = require('webpage').create(); 40 | page.settings.userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'; 41 | 42 | page.open(mimvp_url, {}, function(status) { // http 43 | //page.open(mimvp_url2, {}, function(status) { // socks5 44 | console.log('status : ' + status); 45 | if (status == "success") { 46 | console.log("set proxy success"); 47 | console.log('page.content : ' + page.content); 48 | console.log('page.content length : ' + page.content.length); 49 | } else { 50 | console.log("set proxy fail"); 51 | } 52 | phantom.exit(); 53 | }); 54 | 55 | // 执行命令： 56 | // http : phantomjs --proxy-type=http --proxy=23.94.65.132:1080 mimvp-proxy-phantomjs.js 57 | // socks5 : phantomjs --proxy-type=socks5 --proxy=78.63.194.59:35844 --ignore-ssl-errors=true mimvp-proxy-phantomjs.js 58 | -------------------------------------------------------------------------------- /Python2/mimvp-proxy-python2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Python2 支持 http、https、socks4、socks5 5 | # 6 | # 米扑代理示例： 7 | # https://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # https://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2015-11-09 14 | 15 | 16 | import urllib, urllib2 17 | import base64 18 | import socks, socket # 需要引入socks.py文件，请到米扑代理示例下载 19 | 20 | import sys 21 | reload(sys) 22 | sys.setdefaultencoding('utf-8') 23 | 24 | # 全局取消ssl证书验证，防止打开未验证的https网址抛出异常 25 | # urllib2.URLError: 26 | import ssl 27 | ssl._create_default_https_context = ssl._create_unverified_context 28 | 29 | 30 | mimvp_url = "http://proxy.mimvp.com/test_proxy2.php" 31 | mimvp_url2 = "https://proxy.mimvp.com/test_proxy2.php" 32 | 33 | 34 | # 爬取米扑代理API接口 35 | def spider_proxy(): 36 | proxy_url = 'https://proxyapi.mimvp.com/api/fetchopen.php?orderid=863196312334111234' 37 | req = urllib2.Request(proxy_url) 38 | content = urllib2.urlopen(req, timeout=60).read() 39 | proxy_list = content.split("\n") 40 | for proxy in proxy_list: 41 | print proxy 42 | 43 | 44 | 45 | 46 | ################### proxy no auth（代理无用户名密码验证）################### 47 | 48 | proxy_http = {"http" : "http://111.199.144.207:8118"} 49 | proxy_https = {"https" : "http://180.173.111.237:9797"} 50 | proxy_socks4 = {'socks4' : '218.58.52.158:1088'} 51 | proxy_socks5 = {'socks5' : '68.234.190.150:45454'} 52 | 53 | # urllib2 支持 http 54 | def test_http(proxy, mimvp_url): 55 | handler = urllib2.ProxyHandler(proxy) 56 | opener = urllib2.build_opener(handler, urllib2.HTTPHandler) # http 57 | urllib2.install_opener(opener) 58 | req = urllib2.Request(mimvp_url) 59 | content = urllib2.urlopen(req, timeout=60).read() 60 | print len(content), content 61 | opener.close() 62 | 63 | 64 | # urllib2 支持 https 65 | def test_https(proxy, mimvp_url): 66 | handler = urllib2.ProxyHandler(proxy) 67 | opener = urllib2.build_opener(handler, urllib2.HTTPSHandler) # https 68 | urllib2.install_opener(opener) 69 | req = urllib2.Request(mimvp_url) 70 | content = urllib2.urlopen(req, timeout=60).read() 71 | print len(content), content 72 | opener.close() 73 | 74 | 75 | # urllib 支持 http, https 76 | def test_http2(proxy, mimvp_url): 77 | opener = urllib.FancyURLopener(proxy) 78 | f = opener.open(mimvp_url) 79 | content = f.read() 80 | print len(content), content 81 | f.close() 82 | opener.close() 83 | 84 | 85 | # socks4 86 | def test_socks4(socks4, mimvp_url): 87 | socks4_ip = socks4.split(":")[0] 88 | socks4_port = int(socks4.split(":")[1]) 89 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4, socks4_ip, socks4_port) 90 | socket.socket = socks.socksocket 91 | 92 | content = urllib2.urlopen(mimvp_url, timeout=30).read() 93 | print len(content), content 94 | 95 | 96 | # socks5 97 | def test_socks5(socks5, mimvp_url): 98 | socks5_ip = socks5.split(":")[0] 99 | socks5_port = int(socks5.split(":")[1]) 100 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, socks5_ip, socks5_port) 101 | socket.socket = socks.socksocket 102 | 103 | content = urllib2.urlopen(mimvp_url, timeout=30).read() 104 | print len(content), content 105 | 106 | 107 | 108 | 109 | ################### proxy auth（代理有用户名密码验证）################### 110 | 111 | proxy_noauth_http = {"http" : "http://120.24.177.37:3363"} 112 | proxy_noauth_https = {"https" : "http://120.24.177.37:3363"} 113 | 114 | proxy_auth_http = {"http" : "http://username:password@120.24.177.37:3363"} 115 | proxy_auth_https = {"https" : "http://username:password@120.24.177.37:3363"} 116 | proxy_auth_socks5 = {'socks5' : '120.24.177.37:33634'} 117 | 118 | # urllib2 支持 http 119 | def test_auth_http(proxy, mimvp_url): 120 | handler = urllib2.ProxyHandler(proxy) 121 | opener = urllib2.build_opener(handler, urllib2.HTTPHandler) # http 122 | urllib2.install_opener(opener) 123 | req = urllib2.Request(mimvp_url) 124 | content = urllib2.urlopen(req, timeout=60).read() 125 | print len(content), content 126 | opener.close() 127 | 128 | 129 | # urllib2 支持 https 130 | def test_auth_https(proxy, mimvp_url): 131 | handler = urllib2.ProxyHandler(proxy) 132 | opener = urllib2.build_opener(handler, urllib2.HTTPSHandler) # https 133 | urllib2.install_opener(opener) 134 | req = urllib2.Request(mimvp_url) 135 | content = urllib2.urlopen(req, timeout=60).read() 136 | print len(content), content 137 | opener.close() 138 | 139 | 140 | # urllib2 headers (注意：headers认证，只支持访问http网页，不支持访问https网页) 141 | def test_auth_headers(proxy, mimvp_url): 142 | PROXY_USERNAME = 'username' 143 | PROXY_PASSWORD = 'password' 144 | 145 | proxy_type = "" 146 | proxy_ip_port = "" 147 | for proxy_type in proxy: 148 | proxy_value = proxy.get(proxy_type, '') 149 | proxy_ip_port = proxy_value.split("://")[1] 150 | 151 | req = urllib2.Request(mimvp_url) 152 | req.add_header("Proxy-Authorization", "Basic %s" % base64.b64encode(b'%s:%s' % (PROXY_USERNAME, PROXY_PASSWORD))) 153 | req.set_proxy(proxy_ip_port, proxy_type) 154 | content = urllib2.urlopen(req, timeout=60).read() 155 | print len(content), content 156 | 157 | 158 | # urllib 支持 http, https 159 | def test_auth_http2(proxy, mimvp_url): 160 | opener = urllib.FancyURLopener(proxy) 161 | f = opener.open(mimvp_url) 162 | content = f.read() 163 | print len(content), content 164 | f.close() 165 | opener.close() 166 | 167 | 168 | # socks5 169 | def test_auth_socks5(socks5, mimvp_url): 170 | PROXY_USERNAME = 'username' 171 | PROXY_PASSWORD = 'password' 172 | 173 | socks5_ip = socks5.split(":")[0] 174 | socks5_port = int(socks5.split(":")[1]) 175 | rdns = False #是否在代理服务器上进行dns查询 176 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, socks5_ip, socks5_port, rdns, PROXY_USERNAME, PROXY_PASSWORD) 177 | socket.socket = socks.socksocket 178 | 179 | content = urllib2.urlopen(mimvp_url, timeout=30).read() 180 | print len(content), content 181 | 182 | 183 | 184 | 185 | if __name__ == "__main__": 186 | spider_proxy() 187 | 188 | # proxy no auth 189 | test_http(proxy_http, mimvp_url) # http 190 | test_https(proxy_https, mimvp_url2) # https 191 | 192 | test_http2(proxy_http, mimvp_url) # http 193 | test_http2(proxy_https, mimvp_url2) # https 194 | 195 | test_socks4(proxy_socks4['socks4'], mimvp_url) # socks4 196 | test_socks5(proxy_socks5['socks5'], mimvp_url) # socks5 197 | 198 | 199 | # proxy auth 200 | test_auth_http(proxy_auth_http, mimvp_url) # http 201 | test_auth_https(proxy_auth_https, mimvp_url2) # https 202 | 203 | test_auth_headers(proxy_noauth_http, mimvp_url) # http （不支持访问https网页） 204 | 205 | test_auth_http2(proxy_auth_http, mimvp_url) # http 206 | test_auth_http2(proxy_auth_https, mimvp_url2) # https 207 | 208 | test_auth_socks5(proxy_auth_socks5['socks5'], mimvp_url) # socks5 209 | 210 | 211 | -------------------------------------------------------------------------------- /Python2/socks.py: -------------------------------------------------------------------------------- 1 | """SocksiPy - Python SOCKS module. 2 | Version 1.00 3 | 4 | Copyright 2006 Dan-Haim. All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, 7 | are permitted provided that the following conditions are met: 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 3. Neither the name of Dan Haim nor the names of his contributors may be used 14 | to endorse or promote products derived from this software without specific 15 | prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED 18 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 20 | EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA 23 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25 | OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. 26 | 27 | 28 | This module provides a standard socket-like interface for Python 29 | for tunneling connections through SOCKS proxies. 30 | 31 | """ 32 | 33 | import socket 34 | import struct 35 | 36 | PROXY_TYPE_SOCKS4 = 1 37 | PROXY_TYPE_SOCKS5 = 2 38 | PROXY_TYPE_HTTP = 3 39 | 40 | _defaultproxy = None 41 | _orgsocket = socket.socket 42 | 43 | class ProxyError(Exception): 44 | def __init__(self, value): 45 | self.value = value 46 | def __str__(self): 47 | return repr(self.value) 48 | 49 | class GeneralProxyError(ProxyError): 50 | def __init__(self, value): 51 | self.value = value 52 | def __str__(self): 53 | return repr(self.value) 54 | 55 | class Socks5AuthError(ProxyError): 56 | def __init__(self, value): 57 | self.value = value 58 | def __str__(self): 59 | return repr(self.value) 60 | 61 | class Socks5Error(ProxyError): 62 | def __init__(self, value): 63 | self.value = value 64 | def __str__(self): 65 | return repr(self.value) 66 | 67 | class Socks4Error(ProxyError): 68 | def __init__(self, value): 69 | self.value = value 70 | def __str__(self): 71 | return repr(self.value) 72 | 73 | class HTTPError(ProxyError): 74 | def __init__(self, value): 75 | self.value = value 76 | def __str__(self): 77 | return repr(self.value) 78 | 79 | _generalerrors = ("success", 80 | "invalid data", 81 | "not connected", 82 | "not available", 83 | "bad proxy type", 84 | "bad input") 85 | 86 | _socks5errors = ("succeeded", 87 | "general SOCKS server failure", 88 | "connection not allowed by ruleset", 89 | "Network unreachable", 90 | "Host unreachable", 91 | "Connection refused", 92 | "TTL expired", 93 | "Command not supported", 94 | "Address type not supported", 95 | "Unknown error") 96 | 97 | _socks5autherrors = ("succeeded", 98 | "authentication is required", 99 | "all offered authentication methods were rejected", 100 | "unknown username or invalid password", 101 | "unknown error") 102 | 103 | _socks4errors = ("request granted", 104 | "request rejected or failed", 105 | "request rejected because SOCKS server cannot connect to identd on the client", 106 | "request rejected because the client program and identd report different user-ids", 107 | "unknown error") 108 | 109 | def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): 110 | """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) 111 | Sets a default proxy which all further socksocket objects will use, 112 | unless explicitly changed. 113 | """ 114 | global _defaultproxy 115 | _defaultproxy = (proxytype,addr,port,rdns,username,password) 116 | 117 | class socksocket(socket.socket): 118 | """socksocket([family[, type[, proto]]]) -> socket object 119 | 120 | Open a SOCKS enabled socket. The parameters are the same as 121 | those of the standard socket init. In order for SOCKS to work, 122 | you must specify family=AF_INET, type=SOCK_STREAM and proto=0. 123 | """ 124 | 125 | def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): 126 | _orgsocket.__init__(self,family,type,proto,_sock) 127 | if _defaultproxy != None: 128 | self.__proxy = _defaultproxy 129 | else: 130 | self.__proxy = (None, None, None, None, None, None) 131 | self.__proxysockname = None 132 | self.__proxypeername = None 133 | 134 | def __recvall(self, bytes): 135 | """__recvall(bytes) -> data 136 | Receive EXACTLY the number of bytes requested from the socket. 137 | Blocks until the required number of bytes have been received. 138 | """ 139 | data = "" 140 | while len(data) < bytes: 141 | data = data + self.recv(bytes-len(data)) 142 | return data 143 | 144 | def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): 145 | """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) 146 | Sets the proxy to be used. 147 | proxytype - The type of the proxy to be used. Three types 148 | are supported: PROXY_TYPE_SOCKS4 (including socks4a), 149 | PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP 150 | addr - The address of the server (IP or DNS). 151 | port - The port of the server. Defaults to 1080 for SOCKS 152 | servers and 8080 for HTTP proxy servers. 153 | rdns - Should DNS queries be preformed on the remote side 154 | (rather than the local side). The default is True. 155 | Note: This has no effect with SOCKS4 servers. 156 | username - Username to authenticate with to the server. 157 | The default is no authentication. 158 | password - Password to authenticate with to the server. 159 | Only relevant when username is also provided. 160 | """ 161 | self.__proxy = (proxytype,addr,port,rdns,username,password) 162 | 163 | def __negotiatesocks5(self,destaddr,destport): 164 | """__negotiatesocks5(self,destaddr,destport) 165 | Negotiates a connection through a SOCKS5 server. 166 | """ 167 | # First we'll send the authentication packages we support. 168 | if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): 169 | # The username/password details were supplied to the 170 | # setproxy method so we support the USERNAME/PASSWORD 171 | # authentication (in addition to the standard none). 172 | self.sendall("\x05\x02\x00\x02") 173 | else: 174 | # No username/password were entered, therefore we 175 | # only support connections with no authentication. 176 | self.sendall("\x05\x01\x00") 177 | # We'll receive the server's response to determine which 178 | # method was selected 179 | chosenauth = self.__recvall(2) 180 | if chosenauth[0] != "\x05": 181 | self.close() 182 | raise GeneralProxyError((1,_generalerrors[1])) 183 | # Check the chosen authentication method 184 | if chosenauth[1] == "\x00": 185 | # No authentication is required 186 | pass 187 | elif chosenauth[1] == "\x02": 188 | # Okay, we need to perform a basic username/password 189 | # authentication. 190 | self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.proxy[5])) + self.__proxy[5]) 191 | authstat = self.__recvall(2) 192 | if authstat[0] != "\x01": 193 | # Bad response 194 | self.close() 195 | raise GeneralProxyError((1,_generalerrors[1])) 196 | if authstat[1] != "\x00": 197 | # Authentication failed 198 | self.close() 199 | raise Socks5AuthError,((3,_socks5autherrors[3])) 200 | # Authentication succeeded 201 | else: 202 | # Reaching here is always bad 203 | self.close() 204 | if chosenauth[1] == "\xFF": 205 | raise Socks5AuthError((2,_socks5autherrors[2])) 206 | else: 207 | raise GeneralProxyError((1,_generalerrors[1])) 208 | # Now we can request the actual connection 209 | req = "\x05\x01\x00" 210 | # If the given destination address is an IP address, we'll 211 | # use the IPv4 address request even if remote resolving was specified. 212 | try: 213 | ipaddr = socket.inet_aton(destaddr) 214 | req = req + "\x01" + ipaddr 215 | except socket.error: 216 | # Well it's not an IP number, so it's probably a DNS name. 217 | if self.__proxy[3]==True: 218 | # Resolve remotely 219 | ipaddr = None 220 | req = req + "\x03" + chr(len(destaddr)) + destaddr 221 | else: 222 | # Resolve locally 223 | ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) 224 | req = req + "\x01" + ipaddr 225 | req = req + struct.pack(">H",destport) 226 | self.sendall(req) 227 | # Get the response 228 | resp = self.__recvall(4) 229 | if resp[0] != "\x05": 230 | self.close() 231 | raise GeneralProxyError((1,_generalerrors[1])) 232 | elif resp[1] != "\x00": 233 | # Connection failed 234 | self.close() 235 | if ord(resp[1])<=8: 236 | raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])]) 237 | else: 238 | raise Socks5Error(9,_generalerrors[9]) 239 | # Get the bound address/port 240 | elif resp[3] == "\x01": 241 | boundaddr = self.__recvall(4) 242 | elif resp[3] == "\x03": 243 | resp = resp + self.recv(1) 244 | boundaddr = self.__recvall(resp[4]) 245 | else: 246 | self.close() 247 | raise GeneralProxyError((1,_generalerrors[1])) 248 | boundport = struct.unpack(">H",self.__recvall(2))[0] 249 | self.__proxysockname = (boundaddr,boundport) 250 | if ipaddr != None: 251 | self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) 252 | else: 253 | self.__proxypeername = (destaddr,destport) 254 | 255 | def getproxysockname(self): 256 | """getsockname() -> address info 257 | Returns the bound IP address and port number at the proxy. 258 | """ 259 | return self.__proxysockname 260 | 261 | def getproxypeername(self): 262 | """getproxypeername() -> address info 263 | Returns the IP and port number of the proxy. 264 | """ 265 | return _orgsocket.getpeername(self) 266 | 267 | def getpeername(self): 268 | """getpeername() -> address info 269 | Returns the IP address and port number of the destination 270 | machine (note: getproxypeername returns the proxy) 271 | """ 272 | return self.__proxypeername 273 | 274 | def __negotiatesocks4(self,destaddr,destport): 275 | """__negotiatesocks4(self,destaddr,destport) 276 | Negotiates a connection through a SOCKS4 server. 277 | """ 278 | # Check if the destination address provided is an IP address 279 | rmtrslv = False 280 | try: 281 | ipaddr = socket.inet_aton(destaddr) 282 | except socket.error: 283 | # It's a DNS name. Check where it should be resolved. 284 | if self.__proxy[3]==True: 285 | ipaddr = "\x00\x00\x00\x01" 286 | rmtrslv = True 287 | else: 288 | ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) 289 | # Construct the request packet 290 | req = "\x04\x01" + struct.pack(">H",destport) + ipaddr 291 | # The username parameter is considered userid for SOCKS4 292 | if self.__proxy[4] != None: 293 | req = req + self.__proxy[4] 294 | req = req + "\x00" 295 | # DNS name if remote resolving is required 296 | # NOTE: This is actually an extension to the SOCKS4 protocol 297 | # called SOCKS4A and may not be supported in all cases. 298 | if rmtrslv==True: 299 | req = req + destaddr + "\x00" 300 | self.sendall(req) 301 | # Get the response from the server 302 | resp = self.__recvall(8) 303 | if resp[0] != "\x00": 304 | # Bad data 305 | self.close() 306 | raise GeneralProxyError((1,_generalerrors[1])) 307 | if resp[1] != "\x5A": 308 | # Server returned an error 309 | self.close() 310 | if ord(resp[1]) in (91,92,93): 311 | self.close() 312 | raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) 313 | else: 314 | raise Socks4Error((94,_socks4errors[4])) 315 | # Get the bound address/port 316 | self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) 317 | if rmtrslv != None: 318 | self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) 319 | else: 320 | self.__proxypeername = (destaddr,destport) 321 | 322 | def __negotiatehttp(self,destaddr,destport): 323 | """__negotiatehttp(self,destaddr,destport) 324 | Negotiates a connection through an HTTP server. 325 | """ 326 | # If we need to resolve locally, we do this now 327 | if self.__proxy[3] == False: 328 | addr = socket.gethostbyname(destaddr) 329 | else: 330 | addr = destaddr 331 | self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") 332 | # We read the response until we get the string "\r\n\r\n" 333 | resp = self.recv(1) 334 | while resp.find("\r\n\r\n")==-1: 335 | resp = resp + self.recv(1) 336 | # We just need the first line to check if the connection 337 | # was successful 338 | statusline = resp.splitlines()[0].split(" ",2) 339 | if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): 340 | self.close() 341 | raise GeneralProxyError((1,_generalerrors[1])) 342 | try: 343 | statuscode = int(statusline[1]) 344 | except ValueError: 345 | self.close() 346 | raise GeneralProxyError((1,_generalerrors[1])) 347 | if statuscode != 200: 348 | self.close() 349 | raise HTTPError((statuscode,statusline[2])) 350 | self.__proxysockname = ("0.0.0.0",0) 351 | self.__proxypeername = (addr,destport) 352 | 353 | def connect(self,destpair): 354 | """connect(self,despair) 355 | Connects to the specified destination through a proxy. 356 | destpar - A tuple of the IP/DNS address and the port number. 357 | (identical to socket's connect). 358 | To select the proxy server use setproxy(). 359 | """ 360 | # Do a minimal input check first 361 | if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): 362 | raise GeneralProxyError((5,_generalerrors[5])) 363 | if self.__proxy[0] == PROXY_TYPE_SOCKS5: 364 | if self.__proxy[2] != None: 365 | portnum = self.__proxy[2] 366 | else: 367 | portnum = 1080 368 | _orgsocket.connect(self,(self.__proxy[1],portnum)) 369 | self.__negotiatesocks5(destpair[0],destpair[1]) 370 | elif self.__proxy[0] == PROXY_TYPE_SOCKS4: 371 | if self.__proxy[2] != None: 372 | portnum = self.__proxy[2] 373 | else: 374 | portnum = 1080 375 | _orgsocket.connect(self,(self.__proxy[1],portnum)) 376 | self.__negotiatesocks4(destpair[0],destpair[1]) 377 | elif self.__proxy[0] == PROXY_TYPE_HTTP: 378 | if self.__proxy[2] != None: 379 | portnum = self.__proxy[2] 380 | else: 381 | portnum = 8080 382 | _orgsocket.connect(self,(self.__proxy[1],portnum)) 383 | self.__negotiatehttp(destpair[0],destpair[1]) 384 | elif self.__proxy[0] == None: 385 | _orgsocket.connect(self,(destpair[0],destpair[1])) 386 | else: 387 | raise GeneralProxyError((4,_generalerrors[4])) 388 | -------------------------------------------------------------------------------- /Python2/socks.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Python2/socks.pyc -------------------------------------------------------------------------------- /Python3/mimvp-proxy-python3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Python3 支持 http、https、socks4、socks5 5 | # 6 | # Python3使用socks4/5，需安装PySocks 7 | # pip3 install PySocks 8 | # 9 | # 米扑代理示例： 10 | # https://proxy.mimvp.com/demo2.php 11 | # 12 | # 米扑代理购买： 13 | # https://proxy.mimvp.com 14 | # 15 | # mimvp.com 16 | # 2016-10-22 17 | 18 | 19 | import urllib.request # Python3将urllib和urllib2合二为一，并重组了下包结构 20 | import socket 21 | import socks # 安装 pip3 install PySocks 22 | 23 | 24 | mimvp_url = "http://proxy.mimvp.com/test_proxy2.php" 25 | mimvp_url2 = "https://proxy.mimvp.com/test_proxy2.php" 26 | 27 | 28 | # 全局取消ssl证书验证，防止打开未验证的https网址抛出异常 29 | # urllib.error.URLError: 30 | import ssl 31 | ssl._create_default_https_context = ssl._create_unverified_context 32 | 33 | 34 | # 通过API提供米扑代理 35 | def spider_proxy(): 36 | proxy_url = 'https://proxyapi.mimvp.com/api/fetchsecret.php?orderid=863196312334111234&http_type=3' 37 | req = urllib.request.Request(proxy_url) 38 | content = urllib.request.urlopen(req, timeout=30).read() 39 | proxy_list = content.decode().split("\n") 40 | for proxy in proxy_list: 41 | print(proxy) 42 | 43 | 44 | 45 | ################### proxy no auth（代理无用户名密码验证）################### 46 | 47 | proxy_http = {"http" : "http://47.94.230.42:9999"} 48 | proxy_https = {"https" : "http://221.180.204.144:80"} 49 | proxy_socks4 = {'socks4': '61.234.37.3:1080'} 50 | proxy_socks5 = {'socks5': '183.156.71.161:1080'} 51 | 52 | # urllib 支持 http, https 53 | def test_http(proxy, mimvp_url): 54 | socket.setdefaulttimeout(60) 55 | handler = urllib.request.ProxyHandler(proxy) 56 | opener = urllib.request.build_opener(handler) 57 | urllib.request.install_opener(opener) 58 | content = urllib.request.urlopen(mimvp_url, timeout=60).read().decode("utf8") 59 | print(len(content), content) 60 | opener.close() 61 | 62 | 63 | # socks4 64 | def test_socks4(socks4, mimvp_url): 65 | socks4_ip = socks4.split(":")[0] 66 | socks4_port = int(socks4.split(":")[1]) 67 | socks.set_default_proxy(socks.SOCKS4, socks4_ip, socks4_port) 68 | socket.socket = socks.socksocket 69 | content = urllib.request.urlopen(mimvp_url, timeout=30).read().decode("utf8") 70 | print(len(content), content) 71 | 72 | 73 | # socks5 74 | def test_socks5(socks5, mimvp_url): 75 | socks5_ip = socks5.split(":")[0] 76 | socks5_port = int(socks5.split(":")[1]) 77 | socks.set_default_proxy(socks.SOCKS5, socks5_ip, socks5_port) 78 | socket.socket = socks.socksocket 79 | content = urllib.request.urlopen(mimvp_url, timeout=30).read().decode("utf8") 80 | print(len(content), content) 81 | 82 | 83 | 84 | 85 | ################### proxy auth（代理有用户名密码验证）################### 86 | 87 | proxy_noauth_http = {"http" : "http://120.24.77.37:1056"} 88 | proxy_noauth_https = {"https" : "http://120.24.77.37:1056"} 89 | proxy_auth_socks5 = {'socks5' : '120.24.177.37:10545'} 90 | 91 | proxy_auth_http = {"http" : "http://username:password@120.24.77.37:1056"} 92 | proxy_auth_https = {"https" : "http://username:password@120.24.77.37:1056"} 93 | 94 | # urllib 支持 http, https 95 | def test_auth_http(proxy, mimvp_url): 96 | socket.setdefaulttimeout(30) 97 | handler = urllib.request.ProxyHandler(proxy) 98 | opener = urllib.request.build_opener(handler) 99 | urllib.request.install_opener(opener) 100 | content = urllib.request.urlopen(mimvp_url, timeout=60).read().decode("utf8") 101 | print(len(content), content) 102 | opener.close() 103 | 104 | 105 | # socks5 106 | def test_auth_socks5(socks5, mimvp_url): 107 | PROXY_USERNAME = 'username' 108 | PROXY_PASSWORD = 'password' 109 | 110 | socks5_ip = socks5.split(":")[0] 111 | socks5_port = int(socks5.split(":")[1]) 112 | rdns = False # 是否在代理服务器上进行dns查询 113 | socks.set_default_proxy(socks.SOCKS5, socks5_ip, socks5_port, rdns, PROXY_USERNAME, PROXY_PASSWORD) 114 | socket.socket = socks.socksocket 115 | content = urllib.request.urlopen(mimvp_url, timeout=30).read().decode("utf8") 116 | print(len(content), content) 117 | 118 | 119 | 120 | if __name__ == "__main__": 121 | spider_proxy() # 通过API获取米扑代理 122 | 123 | # proxy no auth 124 | test_http(proxy_http, mimvp_url) # http 125 | test_http(proxy_https, mimvp_url2) # https 126 | 127 | test_socks4(proxy_socks4['socks4'], mimvp_url) # socks4 访问http网页 128 | test_socks4(proxy_socks4['socks4'], mimvp_url2) # socks4 访问https网页 129 | 130 | test_socks5(proxy_socks5['socks5'], mimvp_url) # socks5 访问http网页 131 | test_socks5(proxy_socks5['socks5'], mimvp_url2) # socks5 访问https网页 132 | 133 | 134 | # proxy auth 135 | test_auth_http(proxy_auth_http, mimvp_url) # http 136 | test_auth_http(proxy_auth_https, mimvp_url2) # https 137 | 138 | test_auth_socks5(proxy_auth_socks5['socks5'], mimvp_url) # socks5 访问http网页 139 | test_auth_socks5(proxy_auth_socks5['socks5'], mimvp_url2) # socks5 访问https网页 140 | -------------------------------------------------------------------------------- /PythonPyspider/mimvp-proxy-python-pyspider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | # Project: mimvp_proxy_pyspider 4 | # 5 | # Python PySpider 支持 http、https 6 | # 7 | # 米扑代理示例： 8 | # http://proxy.mimvp.com/demo2.php 9 | # 10 | # 米扑代理购买： 11 | # http://proxy.mimvp.com 12 | # 13 | # mimvp.com 14 | # 2017-07-22 15 | # 16 | # 米扑博客示例详解 : 17 | # http://blog.mimvp.com/2017/08/python-pyspider-an-zhuang-yu-kai-fa/ 18 | 19 | 20 | ############ 方式1：pyspider crawl_config ############ 21 | 22 | from pyspider.libs.base_handler import * 23 | 24 | class Handler(BaseHandler): 25 | crawl_config = { 26 | 'proxy' : 'http://188.226.141.217:8080', # http 27 | 'proxy' : 'https://182.253.32.65:3128' # https 28 | } 29 | 30 | @every(minutes=24 * 60) 31 | def on_start(self): 32 | self.crawl('http://proxy.mimvp.com/exist.php', callback=self.index_page) 33 | 34 | @config(age=10 * 24 * 60 * 60) 35 | def index_page(self, response): 36 | for each in response.doc('a[href^="http"]').items(): 37 | self.crawl(each.attr.href, callback=self.detail_page) 38 | 39 | @config(priority=2) 40 | def detail_page(self, response): 41 | return { 42 | "url": response.url, 43 | "title": response.doc('title').text(), 44 | } 45 | 46 | 47 | 48 | ############ 方式2：pyspider --phantomjs-proxy 启动 ############ 49 | 50 | # $ pyspider --help 51 | # Usage: pyspider [OPTIONS] COMMAND [ARGS]... 52 | # 53 | # A powerful spider system in python. 54 | # 55 | # Options: 56 | # -c, --config FILENAME a json file with default values for 57 | # subcommands. {"webui": {"port":5001}} 58 | # --logging-config TEXT logging config file for built-in python 59 | # logging module [default: /Library/Framework 60 | # s/Python.framework/Versions/2.7/lib/python2. 61 | # 7/site-packages/pyspider/logging.conf] 62 | # --debug debug mode 63 | # --queue-maxsize INTEGER maxsize of queue 64 | # --taskdb TEXT database url for taskdb, default: sqlite 65 | # --projectdb TEXT database url for projectdb, default: sqlite 66 | # --resultdb TEXT database url for resultdb, default: sqlite 67 | # --message-queue TEXT connection url to message queue, default: 68 | # builtin multiprocessing.Queue 69 | # --amqp-url TEXT [deprecated] amqp url for rabbitmq. please 70 | # use --message-queue instead. 71 | # --beanstalk TEXT [deprecated] beanstalk config for beanstalk 72 | # queue. please use --message-queue instead. 73 | # --phantomjs-proxy TEXT phantomjs proxy ip:port 74 | # --data-path TEXT data dir path 75 | # --add-sys-path / --not-add-sys-path 76 | # add current working directory to python lib 77 | # search path 78 | # --version Show the version and exit. 79 | # --help Show this message and exit. 80 | 81 | pyspider --phantomjs-proxy "188.226.141.217:8080" all 82 | -------------------------------------------------------------------------------- /PythonRequests/mimvp-proxy-python-requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | # 4 | # Python requests 支持 http、https、socks4、socks5 5 | # 6 | # requests不是Python原生库，需要安装： 7 | # pip install requests 8 | # pip install requests[socks] # requests >= 2.10.0 9 | # 10 | # 米扑代理示例： 11 | # https://proxy.mimvp.com/demo2.php 12 | # 13 | # 米扑代理购买： 14 | # https://proxy.mimvp.com 15 | # 16 | # mimvp.com 17 | # 2016-09-16 18 | 19 | 20 | import requests 21 | import base64 22 | import socks, socket # 需要引入socks.py文件，请到米扑代理示例下载 23 | 24 | # 用于无用户名密码且访问https网址的代理 25 | # import ssl 26 | # ssl._create_default_https_context = ssl._create_unverified_context 27 | 28 | 29 | mimvp_url = "http://proxy.mimvp.com/test_proxy2.php" # http 30 | mimvp_url2 = "https://proxy.mimvp.com/test_proxy2.php" # https 31 | 32 | 33 | ## proxy no auth （代理无用户名密码验证） 34 | # 使用代理 http, https 35 | proxies = { 36 | "http" : "http://91.121.162.173:80", 37 | "https" : "http://190.24.131.250:3128", 38 | } 39 | 40 | req = requests.get(mimvp_url, proxies=proxies, timeout=30) # http 41 | print("mimvp text : " + req.text) 42 | 43 | req = requests.get(mimvp_url2, proxies=proxies, timeout=30, verify=False) # https 44 | print("mimvp text : " + req.text) 45 | 46 | 47 | 48 | # 使用代理 socks4 49 | proxies = { 50 | 'socks4' : '222.83.247.90:1080', 51 | } 52 | 53 | socks4_ip = proxies['socks4'].split(":")[0] 54 | socks4_port = int(proxies['socks4'].split(":")[1]) 55 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4, socks4_ip, socks4_port) 56 | socket.socket = socks.socksocket 57 | 58 | req = requests.get(mimvp_url, timeout=30) # http 59 | print("mimvp text : " + req.text) 60 | 61 | req = requests.get(mimvp_url2, timeout=30, verify=False) # https 62 | print("mimvp text : " + req.text) 63 | 64 | 65 | 66 | # 方式1：使用代理 socks5 67 | proxies = { 68 | 'socks5' : '121.40.195.151:9999', 69 | } 70 | 71 | socks5_ip = proxies['socks5'].split(":")[0] 72 | socks5_port = int(proxies['socks5'].split(":")[1]) 73 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, socks5_ip, socks5_port) 74 | socket.socket = socks.socksocket 75 | 76 | req = requests.get(mimvp_url, timeout=30) # http 77 | print("mimvp text : " + req.text) 78 | 79 | req = requests.get(mimvp_url2, timeout=30, verify=False) # https 80 | print("mimvp text : " + req.text) 81 | 82 | 83 | 84 | # 方式2：使用代理 socks5 (需要安装 pip install requests[socks] 且要求 requests >= 2.10.0) 85 | proxies = { 86 | "http" : "socks5://59.78.45.101:1080", 87 | "https" : "socks5://59.78.45.101:1080", 88 | } 89 | 90 | req = requests.get(mimvp_url, proxies=proxies, timeout=30) # http 91 | print("mimvp text : " + req.text) 92 | 93 | req = requests.get(mimvp_url2, proxies=proxies, timeout=30, verify=False) # https 94 | print("mimvp text : " + req.text) 95 | 96 | 97 | 98 | 99 | ## proxy auth （代理有用户名密码验证） 100 | # 方式1：使用代理 http, https 101 | proxies = { 102 | "http" : "http://username:password@139.199.22.226:4765", 103 | "https" : "http://username:password@139.199.22.226:4765", 104 | } 105 | 106 | req = requests.get(mimvp_url, proxies=proxies, timeout=30) # http 107 | print("mimvp text : " + req.text) 108 | 109 | req = requests.get(mimvp_url2, proxies=proxies, timeout=30) # https 110 | print("mimvp text : " + req.text) 111 | 112 | 113 | 114 | # 方式2：使用代理 http (注意：headers认证，只支持访问http网页，不支持访问https网页) 115 | PROXY_USERNAME = 'username' 116 | PROXY_PASSWORD = 'password' 117 | 118 | proxies = { 119 | "http" : "http://111.230.100.182:3712", 120 | "https" : "http://111.230.100.182:3712", 121 | } 122 | 123 | headers = { 124 | "Host" : "proxy.mimvp.com", 125 | "User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", 126 | "Accept-Encoding" : "Gzip", 127 | "Proxy-Authorization" : "Basic %s" % base64.b64encode(b'%s:%s' % (PROXY_USERNAME, PROXY_PASSWORD)), 128 | } 129 | 130 | req = requests.get(mimvp_url, proxies=proxies, headers=headers, timeout=30) # http (经米扑验证，此方式不支持https) 131 | print("mimvp text : " + req.text) 132 | 133 | 134 | 135 | # 使用代理 socks5 (需要安装 pip install requests[socks] 且要求 requests >= 2.10.0) 136 | proxies = { 137 | "http" : "socks5://username:password@123.53.118.23:1872", 138 | "https" : "socks5://username:password@123.53.118.23:1872", 139 | } 140 | 141 | req = requests.get(mimvp_url, proxies=proxies, timeout=30) # http 142 | print("mimvp text : " + req.text) 143 | 144 | req = requests.get(mimvp_url2, proxies=proxies, timeout=30, verify=False) # https 145 | print("mimvp text : " + req.text) 146 | 147 | 148 | 149 | 150 | ################### requests 简单示例 ################### 151 | 152 | # 实时获取米扑代理API接口 153 | mimvp_proxy_url = 'https://proxyapi.mimvp.com/api/fetchopen.php?orderid=863196322034111234' 154 | req = requests.get(url=mimvp_proxy_url, timeout=30) 155 | req_text = req.text 156 | proxy_list = req_text.split("\n") 157 | for proxy in proxy_list: 158 | print proxy 159 | 160 | 161 | # 爬取米扑科技首页 162 | req = requests.get(url = 'https://mimvp.com') 163 | print("status_code : " + str(req.status_code)) 164 | print("mimvp text : " + req.text) 165 | 166 | 167 | # 爬取米扑代理（含请求参数） 168 | req = requests.get(url='https://proxy.mimvp.com/free.php', params={'proxy':'out_tp','sort':'p_ping'}) 169 | print("status_code : " + str(req.status_code)) 170 | print("mimvp text : " + req.text) -------------------------------------------------------------------------------- /PythonRequests/socks.py: -------------------------------------------------------------------------------- 1 | """SocksiPy - Python SOCKS module. 2 | Version 1.00 3 | 4 | Copyright 2006 Dan-Haim. All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, 7 | are permitted provided that the following conditions are met: 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 3. Neither the name of Dan Haim nor the names of his contributors may be used 14 | to endorse or promote products derived from this software without specific 15 | prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED 18 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 20 | EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA 23 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25 | OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. 26 | 27 | 28 | This module provides a standard socket-like interface for Python 29 | for tunneling connections through SOCKS proxies. 30 | 31 | """ 32 | 33 | import socket 34 | import struct 35 | 36 | PROXY_TYPE_SOCKS4 = 1 37 | PROXY_TYPE_SOCKS5 = 2 38 | PROXY_TYPE_HTTP = 3 39 | 40 | _defaultproxy = None 41 | _orgsocket = socket.socket 42 | 43 | class ProxyError(Exception): 44 | def __init__(self, value): 45 | self.value = value 46 | def __str__(self): 47 | return repr(self.value) 48 | 49 | class GeneralProxyError(ProxyError): 50 | def __init__(self, value): 51 | self.value = value 52 | def __str__(self): 53 | return repr(self.value) 54 | 55 | class Socks5AuthError(ProxyError): 56 | def __init__(self, value): 57 | self.value = value 58 | def __str__(self): 59 | return repr(self.value) 60 | 61 | class Socks5Error(ProxyError): 62 | def __init__(self, value): 63 | self.value = value 64 | def __str__(self): 65 | return repr(self.value) 66 | 67 | class Socks4Error(ProxyError): 68 | def __init__(self, value): 69 | self.value = value 70 | def __str__(self): 71 | return repr(self.value) 72 | 73 | class HTTPError(ProxyError): 74 | def __init__(self, value): 75 | self.value = value 76 | def __str__(self): 77 | return repr(self.value) 78 | 79 | _generalerrors = ("success", 80 | "invalid data", 81 | "not connected", 82 | "not available", 83 | "bad proxy type", 84 | "bad input") 85 | 86 | _socks5errors = ("succeeded", 87 | "general SOCKS server failure", 88 | "connection not allowed by ruleset", 89 | "Network unreachable", 90 | "Host unreachable", 91 | "Connection refused", 92 | "TTL expired", 93 | "Command not supported", 94 | "Address type not supported", 95 | "Unknown error") 96 | 97 | _socks5autherrors = ("succeeded", 98 | "authentication is required", 99 | "all offered authentication methods were rejected", 100 | "unknown username or invalid password", 101 | "unknown error") 102 | 103 | _socks4errors = ("request granted", 104 | "request rejected or failed", 105 | "request rejected because SOCKS server cannot connect to identd on the client", 106 | "request rejected because the client program and identd report different user-ids", 107 | "unknown error") 108 | 109 | def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): 110 | """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) 111 | Sets a default proxy which all further socksocket objects will use, 112 | unless explicitly changed. 113 | """ 114 | global _defaultproxy 115 | _defaultproxy = (proxytype,addr,port,rdns,username,password) 116 | 117 | class socksocket(socket.socket): 118 | """socksocket([family[, type[, proto]]]) -> socket object 119 | 120 | Open a SOCKS enabled socket. The parameters are the same as 121 | those of the standard socket init. In order for SOCKS to work, 122 | you must specify family=AF_INET, type=SOCK_STREAM and proto=0. 123 | """ 124 | 125 | def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): 126 | _orgsocket.__init__(self,family,type,proto,_sock) 127 | if _defaultproxy != None: 128 | self.__proxy = _defaultproxy 129 | else: 130 | self.__proxy = (None, None, None, None, None, None) 131 | self.__proxysockname = None 132 | self.__proxypeername = None 133 | 134 | def __recvall(self, bytes): 135 | """__recvall(bytes) -> data 136 | Receive EXACTLY the number of bytes requested from the socket. 137 | Blocks until the required number of bytes have been received. 138 | """ 139 | data = "" 140 | while len(data) < bytes: 141 | data = data + self.recv(bytes-len(data)) 142 | return data 143 | 144 | def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): 145 | """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) 146 | Sets the proxy to be used. 147 | proxytype - The type of the proxy to be used. Three types 148 | are supported: PROXY_TYPE_SOCKS4 (including socks4a), 149 | PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP 150 | addr - The address of the server (IP or DNS). 151 | port - The port of the server. Defaults to 1080 for SOCKS 152 | servers and 8080 for HTTP proxy servers. 153 | rdns - Should DNS queries be preformed on the remote side 154 | (rather than the local side). The default is True. 155 | Note: This has no effect with SOCKS4 servers. 156 | username - Username to authenticate with to the server. 157 | The default is no authentication. 158 | password - Password to authenticate with to the server. 159 | Only relevant when username is also provided. 160 | """ 161 | self.__proxy = (proxytype,addr,port,rdns,username,password) 162 | 163 | def __negotiatesocks5(self,destaddr,destport): 164 | """__negotiatesocks5(self,destaddr,destport) 165 | Negotiates a connection through a SOCKS5 server. 166 | """ 167 | # First we'll send the authentication packages we support. 168 | if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): 169 | # The username/password details were supplied to the 170 | # setproxy method so we support the USERNAME/PASSWORD 171 | # authentication (in addition to the standard none). 172 | self.sendall("\x05\x02\x00\x02") 173 | else: 174 | # No username/password were entered, therefore we 175 | # only support connections with no authentication. 176 | self.sendall("\x05\x01\x00") 177 | # We'll receive the server's response to determine which 178 | # method was selected 179 | chosenauth = self.__recvall(2) 180 | if chosenauth[0] != "\x05": 181 | self.close() 182 | raise GeneralProxyError((1,_generalerrors[1])) 183 | # Check the chosen authentication method 184 | if chosenauth[1] == "\x00": 185 | # No authentication is required 186 | pass 187 | elif chosenauth[1] == "\x02": 188 | # Okay, we need to perform a basic username/password 189 | # authentication. 190 | self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.proxy[5])) + self.__proxy[5]) 191 | authstat = self.__recvall(2) 192 | if authstat[0] != "\x01": 193 | # Bad response 194 | self.close() 195 | raise GeneralProxyError((1,_generalerrors[1])) 196 | if authstat[1] != "\x00": 197 | # Authentication failed 198 | self.close() 199 | raise Socks5AuthError,((3,_socks5autherrors[3])) 200 | # Authentication succeeded 201 | else: 202 | # Reaching here is always bad 203 | self.close() 204 | if chosenauth[1] == "\xFF": 205 | raise Socks5AuthError((2,_socks5autherrors[2])) 206 | else: 207 | raise GeneralProxyError((1,_generalerrors[1])) 208 | # Now we can request the actual connection 209 | req = "\x05\x01\x00" 210 | # If the given destination address is an IP address, we'll 211 | # use the IPv4 address request even if remote resolving was specified. 212 | try: 213 | ipaddr = socket.inet_aton(destaddr) 214 | req = req + "\x01" + ipaddr 215 | except socket.error: 216 | # Well it's not an IP number, so it's probably a DNS name. 217 | if self.__proxy[3]==True: 218 | # Resolve remotely 219 | ipaddr = None 220 | req = req + "\x03" + chr(len(destaddr)) + destaddr 221 | else: 222 | # Resolve locally 223 | ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) 224 | req = req + "\x01" + ipaddr 225 | req = req + struct.pack(">H",destport) 226 | self.sendall(req) 227 | # Get the response 228 | resp = self.__recvall(4) 229 | if resp[0] != "\x05": 230 | self.close() 231 | raise GeneralProxyError((1,_generalerrors[1])) 232 | elif resp[1] != "\x00": 233 | # Connection failed 234 | self.close() 235 | if ord(resp[1])<=8: 236 | raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])]) 237 | else: 238 | raise Socks5Error(9,_generalerrors[9]) 239 | # Get the bound address/port 240 | elif resp[3] == "\x01": 241 | boundaddr = self.__recvall(4) 242 | elif resp[3] == "\x03": 243 | resp = resp + self.recv(1) 244 | boundaddr = self.__recvall(resp[4]) 245 | else: 246 | self.close() 247 | raise GeneralProxyError((1,_generalerrors[1])) 248 | boundport = struct.unpack(">H",self.__recvall(2))[0] 249 | self.__proxysockname = (boundaddr,boundport) 250 | if ipaddr != None: 251 | self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) 252 | else: 253 | self.__proxypeername = (destaddr,destport) 254 | 255 | def getproxysockname(self): 256 | """getsockname() -> address info 257 | Returns the bound IP address and port number at the proxy. 258 | """ 259 | return self.__proxysockname 260 | 261 | def getproxypeername(self): 262 | """getproxypeername() -> address info 263 | Returns the IP and port number of the proxy. 264 | """ 265 | return _orgsocket.getpeername(self) 266 | 267 | def getpeername(self): 268 | """getpeername() -> address info 269 | Returns the IP address and port number of the destination 270 | machine (note: getproxypeername returns the proxy) 271 | """ 272 | return self.__proxypeername 273 | 274 | def __negotiatesocks4(self,destaddr,destport): 275 | """__negotiatesocks4(self,destaddr,destport) 276 | Negotiates a connection through a SOCKS4 server. 277 | """ 278 | # Check if the destination address provided is an IP address 279 | rmtrslv = False 280 | try: 281 | ipaddr = socket.inet_aton(destaddr) 282 | except socket.error: 283 | # It's a DNS name. Check where it should be resolved. 284 | if self.__proxy[3]==True: 285 | ipaddr = "\x00\x00\x00\x01" 286 | rmtrslv = True 287 | else: 288 | ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) 289 | # Construct the request packet 290 | req = "\x04\x01" + struct.pack(">H",destport) + ipaddr 291 | # The username parameter is considered userid for SOCKS4 292 | if self.__proxy[4] != None: 293 | req = req + self.__proxy[4] 294 | req = req + "\x00" 295 | # DNS name if remote resolving is required 296 | # NOTE: This is actually an extension to the SOCKS4 protocol 297 | # called SOCKS4A and may not be supported in all cases. 298 | if rmtrslv==True: 299 | req = req + destaddr + "\x00" 300 | self.sendall(req) 301 | # Get the response from the server 302 | resp = self.__recvall(8) 303 | if resp[0] != "\x00": 304 | # Bad data 305 | self.close() 306 | raise GeneralProxyError((1,_generalerrors[1])) 307 | if resp[1] != "\x5A": 308 | # Server returned an error 309 | self.close() 310 | if ord(resp[1]) in (91,92,93): 311 | self.close() 312 | raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) 313 | else: 314 | raise Socks4Error((94,_socks4errors[4])) 315 | # Get the bound address/port 316 | self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) 317 | if rmtrslv != None: 318 | self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) 319 | else: 320 | self.__proxypeername = (destaddr,destport) 321 | 322 | def __negotiatehttp(self,destaddr,destport): 323 | """__negotiatehttp(self,destaddr,destport) 324 | Negotiates a connection through an HTTP server. 325 | """ 326 | # If we need to resolve locally, we do this now 327 | if self.__proxy[3] == False: 328 | addr = socket.gethostbyname(destaddr) 329 | else: 330 | addr = destaddr 331 | self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") 332 | # We read the response until we get the string "\r\n\r\n" 333 | resp = self.recv(1) 334 | while resp.find("\r\n\r\n")==-1: 335 | resp = resp + self.recv(1) 336 | # We just need the first line to check if the connection 337 | # was successful 338 | statusline = resp.splitlines()[0].split(" ",2) 339 | if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): 340 | self.close() 341 | raise GeneralProxyError((1,_generalerrors[1])) 342 | try: 343 | statuscode = int(statusline[1]) 344 | except ValueError: 345 | self.close() 346 | raise GeneralProxyError((1,_generalerrors[1])) 347 | if statuscode != 200: 348 | self.close() 349 | raise HTTPError((statuscode,statusline[2])) 350 | self.__proxysockname = ("0.0.0.0",0) 351 | self.__proxypeername = (addr,destport) 352 | 353 | def connect(self,destpair): 354 | """connect(self,despair) 355 | Connects to the specified destination through a proxy. 356 | destpar - A tuple of the IP/DNS address and the port number. 357 | (identical to socket's connect). 358 | To select the proxy server use setproxy(). 359 | """ 360 | # Do a minimal input check first 361 | if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): 362 | raise GeneralProxyError((5,_generalerrors[5])) 363 | if self.__proxy[0] == PROXY_TYPE_SOCKS5: 364 | if self.__proxy[2] != None: 365 | portnum = self.__proxy[2] 366 | else: 367 | portnum = 1080 368 | _orgsocket.connect(self,(self.__proxy[1],portnum)) 369 | self.__negotiatesocks5(destpair[0],destpair[1]) 370 | elif self.__proxy[0] == PROXY_TYPE_SOCKS4: 371 | if self.__proxy[2] != None: 372 | portnum = self.__proxy[2] 373 | else: 374 | portnum = 1080 375 | _orgsocket.connect(self,(self.__proxy[1],portnum)) 376 | self.__negotiatesocks4(destpair[0],destpair[1]) 377 | elif self.__proxy[0] == PROXY_TYPE_HTTP: 378 | if self.__proxy[2] != None: 379 | portnum = self.__proxy[2] 380 | else: 381 | portnum = 8080 382 | _orgsocket.connect(self,(self.__proxy[1],portnum)) 383 | self.__negotiatehttp(destpair[0],destpair[1]) 384 | elif self.__proxy[0] == None: 385 | _orgsocket.connect(self,(destpair[0],destpair[1])) 386 | else: 387 | raise GeneralProxyError((4,_generalerrors[4])) 388 | -------------------------------------------------------------------------------- /PythonRequests/socks.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonRequests/socks.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy.zip -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | mimvp_proxy_python_scrapy 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | Default 4 | python 2.7 5 | 6 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//mimvp_proxy_python_scrapy/middlewares.py=utf-8 3 | encoding//mimvp_proxy_python_scrapy/middlewares2.py=utf-8 4 | encoding//mimvp_proxy_python_scrapy/pipelines.py=utf-8 5 | encoding//mimvp_proxy_python_scrapy/settings.py=utf-8 6 | encoding//mimvp_proxy_python_scrapy/spiders/mimvp_spider.py=utf-8 7 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__init__.py -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/middlewares.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/middlewares.cpython-36.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/middlewares2.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/middlewares2.cpython-36.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/settings.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/__pycache__/settings.cpython-36.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | 10 | 11 | class MimvpProxyPythonScrapyItem(scrapy.Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | pass 15 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/middlewares.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Python scrapy 支持 http、https 5 | # 6 | # 米扑代理示例： 7 | # http://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # http://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2009.10.1 14 | 15 | # Python Scrapy 设置代理有两种方式，使用时两种方式选择一种即可 16 | # 方式1： 直接在代码里设置，如 MimvpSpider ——> start_requests 17 | # 方式2： 通过 middlewares.py + settings.py 配置文件设置，步骤： 18 | # 2.1 middlewares.py 添加代理类 ProxyMiddleware，并添加代理 19 | # 2.2 settings.py 开启 DOWNLOADER_MIDDLEWARES，并且添加 'mimvp_proxy_python_scrapy.middlewares.ProxyMiddleware': 100, 20 | 21 | 22 | # Define here the models for your spider middleware 23 | # 24 | # See documentation in: 25 | # http://doc.scrapy.org/en/latest/topics/spider-middleware.html 26 | 27 | from scrapy import signals 28 | 29 | ## 代理设置方式2： 通过 middlewares.py + settings.py 配置文件设置 30 | ## mimvp custom by yourself 31 | class ProxyMiddleware(object): 32 | def process_request(self,request,spider): 33 | 34 | print("middlewares - ProxyMiddleware()") 35 | 36 | # proxy no auth（代理无用户名密码验证，或白名单ip授权） 37 | # 白名单ip授权，请见米扑代理会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 38 | if request.url.startswith("http://"): 39 | request.meta['proxy']="http://140.143.62.84:37746" # http代理 40 | elif request.url.startswith("https://"): 41 | request.meta['proxy']="http://140.143.62.84:37746" # https代理 42 | 43 | # # proxy auth（代理有用户名密码验证） 44 | # # 用户名密码授权，请见米扑代理会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 45 | # if request.url.startswith("http://"): 46 | # request.meta['proxy']="http://username:password@140.143.62.84:37746" # http代理 47 | # elif request.url.startswith("https://"): 48 | # request.meta['proxy']="http://username:password@140.143.62.84:37746" # https代理 49 | 50 | # # proxy authentication （经验证，失败，不推荐） 51 | # proxy_user_pass = "mimvp-guest:welcome2mimvp" 52 | # encoded_user_pass = base64.encodestring(proxy_user_pass) 53 | # request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass 54 | 55 | 56 | 57 | 58 | class MimvpProxyPythonScrapySpiderMiddleware(object): 59 | # Not all methods need to be defined. If a method is not defined, 60 | # scrapy acts as if the spider middleware does not modify the 61 | # passed objects. 62 | 63 | @classmethod 64 | def from_crawler(cls, crawler): 65 | # This method is used by Scrapy to create your spiders. 66 | s = cls() 67 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 68 | return s 69 | 70 | def process_spider_input(self, response, spider): 71 | # Called for each response that goes through the spider 72 | # middleware and into the spider. 73 | 74 | # Should return None or raise an exception. 75 | return None 76 | 77 | def process_spider_output(self, response, result, spider): 78 | # Called with the results returned from the Spider, after 79 | # it has processed the response. 80 | 81 | # Must return an iterable of Request, dict or Item objects. 82 | for i in result: 83 | yield i 84 | 85 | def process_spider_exception(self, response, exception, spider): 86 | # Called when a spider or process_spider_input() method 87 | # (from other spider middleware) raises an exception. 88 | 89 | # Should return either None or an iterable of Response, dict 90 | # or Item objects. 91 | pass 92 | 93 | def process_start_requests(self, start_requests, spider): 94 | # Called with the start requests of the spider, and works 95 | # similarly to the process_spider_output() method, except 96 | # that it doesn’t have a response associated. 97 | 98 | # Must return only requests (not items). 99 | for r in start_requests: 100 | yield r 101 | 102 | def spider_opened(self, spider): 103 | spider.logger.info('Spider opened: %s' % spider.name) 104 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/middlewares2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | from scrapy import signals 9 | 10 | 11 | class MimvpProxyPythonScrapySpiderMiddleware(object): 12 | # Not all methods need to be defined. If a method is not defined, 13 | # scrapy acts as if the spider middleware does not modify the 14 | # passed objects. 15 | 16 | @classmethod 17 | def from_crawler(cls, crawler): 18 | # This method is used by Scrapy to create your spiders. 19 | s = cls() 20 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 21 | return s 22 | 23 | def process_spider_input(self, response, spider): 24 | # Called for each response that goes through the spider 25 | # middleware and into the spider. 26 | 27 | # Should return None or raise an exception. 28 | return None 29 | 30 | def process_spider_output(self, response, result, spider): 31 | # Called with the results returned from the Spider, after 32 | # it has processed the response. 33 | 34 | # Must return an iterable of Request, dict or Item objects. 35 | for i in result: 36 | yield i 37 | 38 | def process_spider_exception(self, response, exception, spider): 39 | # Called when a spider or process_spider_input() method 40 | # (from other spider middleware) raises an exception. 41 | 42 | # Should return either None or an iterable of Response, dict 43 | # or Item objects. 44 | pass 45 | 46 | def process_start_requests(self, start_requests, spider): 47 | # Called with the start requests of the spider, and works 48 | # similarly to the process_spider_output() method, except 49 | # that it doesn’t have a response associated. 50 | 51 | # Must return only requests (not items). 52 | for r in start_requests: 53 | yield r 54 | 55 | def spider_opened(self, spider): 56 | spider.logger.info('Spider opened: %s' % spider.name) 57 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class MimvpProxyPythonScrapyPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for mimvp_proxy_python_scrapy project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html 10 | # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'mimvp_proxy_python_scrapy' 13 | 14 | SPIDER_MODULES = ['mimvp_proxy_python_scrapy.spiders'] 15 | NEWSPIDER_MODULE = 'mimvp_proxy_python_scrapy.spiders' 16 | 17 | 18 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 19 | #USER_AGENT = 'mimvp_proxy_python_scrapy (+http://www.yourdomain.com)' 20 | 21 | # Obey robots.txt rules 22 | ROBOTSTXT_OBEY = True 23 | 24 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 25 | #CONCURRENT_REQUESTS = 32 26 | 27 | # Configure a delay for requests for the same website (default: 0) 28 | # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay 29 | # See also autothrottle settings and docs 30 | #DOWNLOAD_DELAY = 3 31 | # The download delay setting will honor only one of: 32 | #CONCURRENT_REQUESTS_PER_DOMAIN = 16 33 | #CONCURRENT_REQUESTS_PER_IP = 16 34 | 35 | # Disable cookies (enabled by default) 36 | #COOKIES_ENABLED = False 37 | 38 | # Disable Telnet Console (enabled by default) 39 | #TELNETCONSOLE_ENABLED = False 40 | 41 | # Override the default request headers: 42 | #DEFAULT_REQUEST_HEADERS = { 43 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 44 | # 'Accept-Language': 'en', 45 | #} 46 | 47 | # Enable or disable spider middlewares 48 | # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html 49 | #SPIDER_MIDDLEWARES = { 50 | # 'mimvp_proxy_python_scrapy.middlewares.MimvpProxyPythonScrapySpiderMiddleware': 543, 51 | #} 52 | 53 | # Enable or disable downloader middlewares 54 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html 55 | DOWNLOADER_MIDDLEWARES = { 56 | 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110, 57 | 'mimvp_proxy_python_scrapy.middlewares.ProxyMiddleware': 100, ## 注释掉此片段，开启 mimvp_spider - start_requests() 58 | # 'mimvp_proxy_python_scrapy.middlewares.MimvpProxyPythonScrapySpiderMiddleware': 543, 59 | # 'mimvp_proxy_python_scrapy.middlewares.MyCustomDownloaderMiddleware': 543, 60 | } 61 | 62 | # Enable or disable extensions 63 | # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html 64 | #EXTENSIONS = { 65 | # 'scrapy.extensions.telnet.TelnetConsole': None, 66 | #} 67 | 68 | # Configure item pipelines 69 | # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html 70 | #ITEM_PIPELINES = { 71 | # 'mimvp_proxy_python_scrapy.pipelines.MimvpProxyPythonScrapyPipeline': 300, 72 | #} 73 | 74 | # Enable and configure the AutoThrottle extension (disabled by default) 75 | # See http://doc.scrapy.org/en/latest/topics/autothrottle.html 76 | #AUTOTHROTTLE_ENABLED = True 77 | # The initial download delay 78 | #AUTOTHROTTLE_START_DELAY = 5 79 | # The maximum download delay to be set in case of high latencies 80 | #AUTOTHROTTLE_MAX_DELAY = 60 81 | # The average number of requests Scrapy should be sending in parallel to 82 | # each remote server 83 | #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 84 | # Enable showing throttling stats for every response received: 85 | #AUTOTHROTTLE_DEBUG = False 86 | 87 | # Enable and configure HTTP caching (disabled by default) 88 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 89 | #HTTPCACHE_ENABLED = True 90 | #HTTPCACHE_EXPIRATION_SECS = 0 91 | #HTTPCACHE_DIR = 'httpcache' 92 | #HTTPCACHE_IGNORE_HTTP_CODES = [] 93 | #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' 94 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/__pycache__/mimvp_spider.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/__pycache__/mimvp_spider.cpython-36.pyc -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/mimvp_proxy_python_scrapy/spiders/mimvp_spider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Python scrapy 支持 http、https 5 | # 6 | # 米扑代理示例： 7 | # https://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # https://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2016.10.1 14 | 15 | # Python Scrapy 设置代理有两种方式，使用时两种方式选择一种即可 16 | # 方式1： 直接在代码里设置，如 MimvpSpider ——> start_requests 17 | # 方式2： 通过 middlewares.py + settings.py 配置文件设置，步骤： 18 | # 2.1 middlewares.py 添加代理类 ProxyMiddleware，并添加代理 19 | # 2.2 settings.py 开启 DOWNLOADER_MIDDLEWARES，并且添加 'mimvp_proxy_python_scrapy.middlewares.ProxyMiddleware': 100, 20 | # 21 | # 米扑博客想写介绍了 Python scrapy 设置代理的两种方式（附源码） 22 | # https://blog.mimvp.com/article/19083.html 23 | 24 | 25 | import scrapy 26 | 27 | class MimvpSpider(scrapy.spiders.Spider): 28 | name = "mimvp" 29 | allowed_domains = ["mimvp.com"] 30 | start_urls = [ 31 | "http://proxy.mimvp.com/test_proxy2.php", 32 | "https://proxy.mimvp.com/test_proxy2.php", 33 | ] 34 | 35 | 36 | ## 代理设置方式1：直接在代理里设置 37 | ## 注释掉此片段，开启 settings - DOWNLOADER_MIDDLEWARES - 'mimvp_proxy_python_scrapy.middlewares.ProxyMiddleware' 38 | def start_requests(self): 39 | urls = [ 40 | "http://proxy.mimvp.com/test_proxy2.php", 41 | "https://proxy.mimvp.com/test_proxy2.php", 42 | ] 43 | for url in urls: 44 | meta_proxy = "" 45 | 46 | print("mimvp_spider - start_requests()") 47 | 48 | # proxy no auth（代理无用户名密码验证，或白名单ip授权） 49 | # 白名单ip授权，请见米扑代理会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 50 | if url.startswith("http://"): 51 | meta_proxy = "http://140.143.62.84:37746" # http代理 52 | elif url.startswith("https://"): 53 | meta_proxy = "http://140.143.62.84:37746" # https代理 54 | 55 | # # proxy auth（代理有用户名密码验证） 56 | # # 用户名密码授权，请见米扑代理会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 57 | # if url.startswith("http://"): 58 | # meta_proxy = "http://username:password@140.143.62.84:37746" # http代理 59 | # elif url.startswith("https://"): 60 | # meta_proxy = "http://username:password@140.143.62.84:37746" # https代理 61 | 62 | yield scrapy.Request(url=url, callback=self.parse, meta={'proxy': meta_proxy}) 63 | 64 | 65 | def parse(self, response): 66 | mimvp_url = response.url # 爬取时请求的url 67 | body = response.body # 返回网页内容 68 | 69 | print("mimvp_url : " + str(mimvp_url)) 70 | print("body : " + str(body)) 71 | 72 | # unicode_body = response.body_as_unicode() # 返回的html unicode编码 73 | # print("unicode_body : " + str(unicode_body)) 74 | -------------------------------------------------------------------------------- /PythonScrapy/mimvp_proxy_python_scrapy/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.org/en/latest/deploy.html 5 | 6 | [settings] 7 | default = mimvp_proxy_python_scrapy.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = mimvp_proxy_python_scrapy 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 米扑代理示例（mimvp-proxy-demo） 3 | ======== 4 | 5 | 米扑代理示例（mimvp-proxy-demo）聚合了多种编程语言使用代理IP，由北京米扑科技有限公司([mimvp.com](https://mimvp.com))原创分享。 6 | 7 | 米扑代理示例，包含Python、Java、PHP、C#、Go、Perl、Ruby、Shell、NodeJS、PhantomJS、Groovy、Delphi、易语言等十多种编程语言或脚本，通过大量的可运行实例，详细讲解了使用代理IP的正确方法，方便网页爬取、数据采集、自动化测试等领域。 8 | 9 | 米扑代理示例，测试使用的代理IP，全部来自于米扑代理：https://proxy.mimvp.com 10 | 11 | 12 | #### 米扑代理示例官网 : https://proxy.mimvp.com/demo2.php 13 | 14 | 15 |
16 |
17 | 18 | 19 | 编程语言之代理协议 20 | ---------- 21 | 22 | ![代理协议汇总](./cssjs/mimvp-proxy-demo-1-lang-proxy-protocol.png) 23 | 24 | ![编程语言示例](./cssjs/mimvp-proxy-demo-2-lang-proxy-demo.png) 25 | 26 | 27 |
28 |
29 | 30 | 31 | 编程语言之代理示例 32 | ---------- 33 | 34 | #### 1. PHP 设置代理 35 | 36 | ```php 37 | $proxy_http = "http://138.68.165.154:3128"; 38 | $proxy_https = "https://202.53.169.199:3128"; 39 | $proxy_socks4 = "socks4://94.158.70.129:1080"; 40 | $proxy_socks5 = "socks5://173.230.95.147:45454"; 41 | 42 | $mimvp_url = "http://proxy.mimvp.com/exist.php"; 43 | $mimvp_url2 = "https://proxy.mimvp.com/exist.php"; 44 | 45 | // curl 46 | proxy_curl($proxy_http, $mimvp_url); // http 47 | proxy_curl($proxy_https, $mimvp_url); // https 48 | proxy_curl($proxy_socks4, $mimvp_url); // socks4 49 | proxy_curl($proxy_socks5, $mimvp_url); // socks5 50 | 51 | // php curl 支持 http、https、socks4、socks5 52 | function proxy_curl($proxy_uri, $mimvp_url) { 53 | $key = explode('://', $proxy_uri)[0]; // http 54 | $proxy= explode('://', $proxy_uri)[1]; // ip:port 55 | echo "proxy_uri : $proxy_uri; key : $key, proxy : $proxy "; 56 | 57 | $ch = curl_init (); 58 | curl_setopt ( $ch, CURLOPT_URL, $mimvp_url); 59 | curl_setopt ( $ch, CURLOPT_HTTPPROXYTUNNEL, false ); 60 | curl_setopt ( $ch, CURLOPT_PROXY, $proxy ); 61 | 62 | if ($key == "http") { 63 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP ); // http 64 | } 65 | elseif ($key == "https") { 66 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYHOST, 2 ); 67 | curl_setopt ( $ch, CURLOPT_SSL_VERIFYPEER, false ); 68 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTPS ); // https 69 | } 70 | elseif ($key == "socks4") { 71 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4 ); // socks4 72 | } 73 | elseif ($key == "socks5") { 74 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5 ); // socks5 75 | } 76 | else { 77 | curl_setopt ( $ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP ); 78 | } 79 | 80 | curl_setopt ( $ch, CURLOPT_TIMEOUT, 60 ); 81 | curl_setopt ( $ch, CURLOPT_CONNECTTIMEOUT, 60 ); 82 | curl_setopt ( $ch, CURLOPT_HEADER, false ); 83 | curl_setopt ( $ch, CURLOPT_RETURNTRANSFER, true ); // 返回网页内容 84 | 85 | $result = curl_exec ( $ch ); 86 | } 87 | ``` 88 | 89 |
90 | 91 | 92 | #### 2. Python 设置代理 93 | 94 | ```python 95 | proxy_http = {"http":"http://138.68.165.154:3128"} 96 | proxy_https = {"https":"http://191.252.103.93:8080"} 97 | proxy_socks4 = {'socks4': '218.58.52.158:1088'} 98 | proxy_socks5 = {'socks5': '68.234.190.150:45454'} 99 | 100 | mimvp_url = "http://proxy.mimvp.com/exist.php" 101 | mimvp_url2 = "https://proxy.mimvp.com/exist.php" 102 | 103 | # 全局取消ssl证书验证，防止打开未验证的https网址抛出异常 104 | # urllib2.URLError: [urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:590)] 105 | ssl._create_default_https_context = ssl._create_unverified_context 106 | 107 | 108 | # urllib2 支持 http, https 109 | def test_http(proxy, mimvp_url): 110 | handler = urllib2.ProxyHandler(proxy) 111 | opener = urllib2.build_opener(handler) 112 | f = opener.open(mimvp_url, timeout=30) 113 | content = f.read() 114 | print content 115 | print len(content) 116 | f.close() 117 | opener.close() 118 | 119 | 120 | # urllib 支持 http, https 121 | def test_http2(proxy, mimvp_url): 122 | opener = urllib.FancyURLopener(proxy) 123 | f = opener.open(mimvp_url) #### mimvp_url 只能是http网页，不能是https网页 124 | content = f.read() 125 | print content 126 | print len(content) 127 | f.close() 128 | opener.close() 129 | 130 | 131 | # socks4 132 | def test_socks4(socks4, mimvp_url): 133 | socks4_ip = socks4.split(":")[0] 134 | socks4_port = int(socks4.split(":")[1]) 135 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4, socks4_ip, socks4_port) 136 | socket.socket = socks.socksocket 137 | 138 | content = urllib2.urlopen(mimvp_url, timeout=30).read() 139 | print content 140 | print len(content) 141 | 142 | 143 | # socks5 144 | def test_socks5(socks5, mimvp_url): 145 | socks5_ip = socks5.split(":")[0] 146 | socks5_port = int(socks5.split(":")[1]) 147 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, socks5_ip, socks5_port) 148 | socket.socket = socks.socksocket 149 | 150 | content = urllib2.urlopen(mimvp_url, timeout=30).read() 151 | print content 152 | print len(content) 153 | 154 | 155 | if __name__ == "__main__": 156 | # http, https 157 | test_http(proxy_http, mimvp_url) 158 | test_http(proxy_https, mimvp_url2) 159 | 160 | # http 161 | test_http2(proxy_http, mimvp_url) 162 | 163 | # socks4 164 | test_socks4(proxy_socks4['socks4'], mimvp_url) 165 | 166 | # socks5 167 | test_socks5(proxy_socks5['socks5'], mimvp_url) 168 | ``` 169 | 170 |
171 | 172 | 173 | #### 3. Java 设置代理 174 | 175 | ```java 176 | @SuppressWarnings({ "serial" }) 177 | public static HashMap proxyMap = new HashMap() { 178 | { 179 | put("http", "138.68.161.14:3128"); 180 | put("https", "104.236.120.183:8080"); 181 | put("socks4", "113.7.118.112:2346"); 182 | put("socks5", "61.135.155.82:1080"); 183 | } 184 | }; 185 | 186 | final static String proxyUrl = "http://proxy.mimvp.com/exist.php"; 187 | final static String proxyUrl2 = "https://proxy.mimvp.com/exist.php"; 188 | 189 | // 设置系统代理，支持全部协议 http，https，socks4，socks5 190 | private static int proxy_property(String proxyType, String proxyStr) { 191 | int dataLen = 0; 192 | 193 | String proxy_ip = proxyStr.split(":")[0]; 194 | String proxy_port = proxyStr.split(":")[1]; 195 | 196 | Properties prop = System.getProperties(); 197 | 198 | // http 199 | if(proxyType.equals("http")){ 200 | prop.setProperty("http.proxySet", "true"); 201 | prop.setProperty("http.proxyHost", proxy_ip); 202 | prop.setProperty("http.proxyPort", proxy_port); 203 | prop.setProperty("http.nonProxyHosts", "localhost|192.168.0.*"); 204 | } 205 | 206 | // https 207 | if (proxyType.equals("https")) { 208 | prop.setProperty("https.proxyHost", proxy_ip); 209 | prop.setProperty("https.proxyPort", proxy_port); 210 | } 211 | 212 | // socks 213 | if(proxyType.equals("socks4") || proxyType.equals("socks5")){ 214 | prop.setProperty("socksProxySet", "true"); 215 | prop.setProperty("socksProxyHost", proxy_ip); 216 | prop.setProperty("socksProxyPort", proxy_port); 217 | } 218 | 219 | // ftp 220 | if(proxyType.equals("ftp")){ 221 | prop.setProperty("ftp.proxyHost", proxy_ip); 222 | prop.setProperty("ftp.proxyPort", proxy_port); 223 | prop.setProperty("ftp.nonProxyHosts", "localhost|192.168.0.*"); 224 | } 225 | 226 | // // auth 设置登录代理服务器的用户名和密码 227 | // Authenticator.setDefault(new MyAuthenticator("user", "pwd")); 228 | 229 | try{ 230 | URL url = new URL(proxyUrl2); // http://proxy.mimvp.com 231 | URLConnection conn = url.openConnection(); 232 | conn.setConnectTimeout(30 * 1000); 233 | 234 | InputStream in = conn.getInputStream(); 235 | InputStreamReader reader = new InputStreamReader(in); 236 | char[] ch = new char[1024]; 237 | int len = 0; 238 | String data = ""; 239 | while((len = reader.read(ch)) > 0) { 240 | String newData = new String(ch, 0, len); 241 | data += newData; 242 | } 243 | System.out.println("data : " + data); 244 | dataLen = data.length(); 245 | 246 | } catch(Exception e) { 247 | e.printStackTrace(); 248 | } 249 | return dataLen; 250 | } 251 | 252 | static class MyAuthenticator extends Authenticator { 253 | private String user = ""; 254 | private String password = ""; 255 | public MyAuthenticator(String user, String password) { 256 | this.user = user; 257 | this.password = password; 258 | } 259 | protected PasswordAuthentication getPasswordAuthentication() { 260 | return new PasswordAuthentication(user, password.toCharArray()); 261 | } 262 | } 263 | ``` 264 | 265 |
266 | 267 | 268 | #### 4. Shell 设置代理 269 | 270 | ```sh 271 | #!/bin/bash 272 | # 273 | # curl 支持 http、https、socks4、socks5 274 | # wget 支持 http、https 275 | # 276 | # 米扑代理示例： 277 | # https://proxy.mimvp.com/demo2.php 278 | # 279 | # 米扑代理购买： 280 | # https://proxy.mimvp.com 281 | # 282 | # mimvp.com 283 | # 2015-11-09 284 | 285 | 286 | # http代理格式 http_proxy=http://IP:Port 287 | # https代理格式 https_proxy=http://IP:Port 288 | 289 | {'http': 'http://120.77.176.179:8888'} 290 | curl -m 30 --retry 3 -x http://120.77.176.179:8888 http://proxy.mimvp.com/exist.php # http_proxy 291 | wget -T 30 --tries 3 -e "http_proxy=http://120.77.176.179:8888" http://proxy.mimvp.com/exist.php # http_proxy 292 | 293 | {'https': 'http://46.105.214.133:3128'} 294 | curl -m 30 --retry 3 --proxy-insecure -x http://46.105.214.133:3128 -k https://proxy.mimvp.com/exist.php # https_proxy 295 | wget -T 30 --tries 3 --no-check-certificate -e "https_proxy=http://46.105.214.133:3128" https://proxy.mimvp.com/exist.php # https_proxy 296 | 297 | 298 | # curl 支持socks 299 | {'socks4': '101.255.17.145:1080'} 300 | curl -m 30 --retry 3 --socks4 101.255.17.145:1080 http://proxy.mimvp.com/exist.php 301 | 302 | {'socks5': '82.164.233.227:45454'} 303 | curl -m 30 --retry 3 --socks5 82.164.233.227:45454 http://proxy.mimvp.com/exist.php 304 | 305 | # wget 不支持socks 306 | 307 | 308 | ################### wget配置文件设置代理 ################### 309 | 310 | vim ~/.wgetrc 311 | 312 | http_proxy=http://120.77.176.179:8888:8080 313 | https_proxy=http://12.7.17.17:8888:8080 314 | use_proxy = on 315 | wait = 30 316 | 317 | wget -T 30 --tries 3 http://proxy.mimvp.com 318 | 319 | 320 | ################### 设置临时局部代理 ################### 321 | 322 | # proxy no auth 323 | export http_proxy=http://120.77.176.179:8888:8080 324 | export https_proxy=http://12.7.17.17:8888:8080 325 | 326 | # proxy auth 327 | export http_proxy=http://username:password@120.77.176.179:8888:8080 328 | export https_proxy=http://username:password@12.7.17.17:8888:8080 329 | 330 | 331 | # 取消设置 332 | unset http_proxy 333 | unset https_proxy 334 | 335 | 336 | ################### 设置系统全局代理 ################### 337 | 338 | # 修改 /etc/profile，保存并重启服务器 339 | sudo vim /etc/profile # 所有人有效 340 | 或 341 | sudo vim ~/.bashrc # 所有人有效 342 | 或 343 | vim ~/.bash_profile # 个人有效 344 | 345 | 346 | # proxy no auth 347 | export http_proxy=http://120.77.176.179:8888:8080 348 | export https_proxy=http://12.7.17.17:8888:8080 349 | 350 | # proxy auth 351 | export http_proxy=http://username:password@120.77.176.179:8888:8080 352 | export https_proxy=http://username:password@12.7.17.17:8888:8080 353 | 354 | source /etc/profile 355 | 或 356 | source ~/.bashrc 357 | 或 358 | source ~/.bash_profile 359 | 360 | 361 | sudo reboot 362 | ``` 363 | 364 |
365 | 366 | 367 | #### 5. 易语言 设置代理 368 | 369 | ```php 370 | ' 易语言 支持 http 371 | ' 372 | ' 米扑代理示例： 373 | ' https://proxy.mimvp.com/demo2.php 374 | ' 375 | ' 米扑代理购买： 376 | ' https://proxy.mimvp.com 377 | ' 378 | ' mimvp.com 379 | ' 2017-07-28 380 | ' 381 | ' 易语言的安装与开发，请参考米扑博客： 382 | ' https://blog.mimvp.com/2017/08/yi-yu-yan-de-an-zhuang-yu-kai-fa/ 383 | ' 384 | ' 易语言的网络编程和代理示例，请参考米扑博客，附有源代码： 385 | ' https://blog.mimvp.com/2017/08/yi-yu-yan-de-wang-luo-bian-cheng-he-dai-li-shi-li/ 386 | 387 | 388 | ' 变量声明（表格） 389 | .局部变量 代理地址, 文本型 390 | .局部变量 目标网址, 文本型 391 | .局部变量 代理用户名, 文本型 392 | .局部变量 代理密码, 文本型 393 | 394 | 395 | 396 | ' 获取目标网址 397 | .如果 (目标网址编辑框.内容 ＝ “”) 398 | 目标网址 ＝ “http://proxy.mimvp.com/exist.php” 399 | .否则 400 | 目标网址 ＝ 目标网址编辑框.内容 401 | 402 | ' 获取代理地址 403 | .如果 (代理服务器编辑框.内容 ＝ “”) 404 | 代理地址 ＝ “139.59.99.113:8080” 405 | .否则 406 | 代理地址 ＝ 代理服务器编辑框.内容 407 | 408 | .如果结束 409 | 410 | 输出调试文本 (“代理地址： ” ＋ 代理地址) 411 | 412 | ' 设置代理方法1 413 | 网页_置浏览器代理 (代理地址) ' 通过浏览器设置，一般只支持http代理 414 | 超文本浏览框.跳转 (目标网址编辑框.内容, , ) 415 | 416 | 417 | ' 设置代理方法2 418 | 输出调试文本 (网页_访问S (目标网址, , , , , , , , , , 代理地址, , )) 419 | 420 | 421 | ' 设置代理方式3（用户名 +　密码） 422 | 输出调试文本 (网页_访问S (目标网址, , , , , , , , , , 代理地址, 代理用户名, 代理密码)) 423 | ``` 424 | 425 | ![易语言使用代理示例](./cssjs/eyuyan-demo-1.png) 426 | 427 | ![易语言使用代理示例](./cssjs/eyuyan-demo-2.png) 428 | 429 | 430 |
431 |
432 | 433 | 434 | 注意事项 435 | ---------- 436 | 437 | #### 1. 哪儿可找到代理使用示例里的引用包或依赖库？ 438 | 439 | 解答：米扑代理示例的引用包或依赖库，已经放在了mimvp-proxy-demo开源项目里的目录下了，例如：Python2.7 使用socks4/5是引用了socks.py，已经放在了目录下：https://github.com/mimvp/mimvp-proxy-demo/tree/master/Python2/socks.py 440 | 441 | 442 |
443 | 444 | #### 2. 如何安装配置编程语言的开发环境？ 445 | 446 | 解答：米扑代理使用示例，全部由米扑科技编写代码、测试验证，因此米扑科技搭建了全部的开发测试环境，详细配置编程语言的开发环境，请参考米扑博客：[https://blog.mimvp.com](https://blog.mimvp.com) 447 | 448 | * AutoHotkey 安装与开发 449 | * Python scrapy 安装与开发 450 | * Python requests 安装与开发 451 | * Python pyspider 安装与开发 452 | * Python3 urllib 用法详解 453 | * PhantomJS 安装与开发 454 | * PhantomJS Selenium 设置动态代理 455 | * Node.js 安装与开发 456 | * Node.js SuperAgent 安装与开发 457 | * Node.js 设置代理的两种方式：superagent-proxy 和 https-proxy-agent 458 | * Ruby 安装与开发 459 | * Perl 安装与开发 460 | * Delphi 安装与开发 461 | * Groovy 安装与开发 462 | * Scala 安装与开发 463 | * R语言的安装与开发 464 | * Go语言下载、安装、配置、使用 465 | * Go语言支持 http、 https、socks4、socks4a、socks5，以及嵌套代理 466 | * HTTPie 工具使用入门 467 | * 易语言的安装与开发 468 | * 易语言的网络编程和代理示例 469 | 470 | 471 |
472 | 473 | #### 3. 代理使用示例里的代理从哪找？ 474 | 475 | 解答：米扑代理使用示例（mimvp-proxy-demo）中的测试代理IP，全部来自米扑代理[https://proxy.mimvp.com](https://proxy.mimvp.com)，其支持http、https、socks4、socks5等全部协议的代理IP，而且米扑代理覆盖全球120多个国家，中国34个省市，代理非常丰富，为全球代理IP领导品牌，推荐！ 476 | 477 | 478 |
479 |
480 | 481 | 482 | 米扑科技 483 | ---------- 484 | 485 | ![mimvp-logo.png](./cssjs/mimvp-logo.png) 486 | 487 | https://mimvp.com 488 | 489 | -------------------------------------------------------------------------------- /Ruby/mimvp-proxy-ruby.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Ruby 支持 http 4 | # 5 | # 米扑代理示例： 6 | # http://proxy.mimvp.com/demo2.php 7 | # 8 | # 米扑代理购买： 9 | # http://proxy.mimvp.com 10 | # 11 | # mimvp.com 12 | # 2017-03-21 13 | 14 | 15 | require 'rubygems' 16 | require 'net/http' 17 | require 'open-uri' 18 | require 'timeout' 19 | 20 | 21 | mimvp_url = "http://proxy.mimvp.com/exist.php" 22 | mimvp_url2 = "https://proxy.mimvp.com/exist.php" 23 | mimvp_url3 = "https://apps.bdimg.com/libs/jquery-i18n/1.1.1/jquery.i18n.min.js" 24 | 25 | $proxy = '183.222.102.95:8080' 26 | 27 | $proxy_addr = $proxy.split(':')[0].strip 28 | $proxy_port = $proxy.split(':')[1].strip 29 | 30 | puts $proxy_addr 31 | puts $proxy_port 32 | 33 | 34 | begin 35 | Timeout.timeout(30) { 36 | 37 | # mimvp_url = http://proxy.mimvp.com/exist.php 38 | # uri.host = proxy.mimvp.com 39 | # uri.port = 80 40 | # uri.path = /exist.php 41 | uri = URI.parse(mimvp_url) 42 | result = Net::HTTP.new(uri.host, nil, $proxy_addr, $proxy_port).start { |http| 43 | http.get(uri.path) 44 | } 45 | puts result.body 46 | 47 | 48 | # mimvp_url = http://proxy.mimvp.com/exist.php 49 | # uri.host = proxy.mimvp.com 50 | # uri.port = 80 51 | # uri.path = /exist.php 52 | # req = # 53 | uri = URI.parse(mimvp_url) 54 | req = Net::HTTP::Get.new(uri.path) 55 | result = Net::HTTP::Proxy($proxy_addr, $proxy_port).start(uri.host, uri.port) {|http| 56 | http.request(req) 57 | } 58 | puts result.body 59 | 60 | 61 | # # proxy auth （NO TEST） 62 | # $proxy_addr = '' 63 | # $proxy_port = ' e 81 | puts e.inspect 82 | exit 83 | end 84 | 85 | ## 运行命令： ruby mimvp-proxy-ruby.rb 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | #require 'net/http' 95 | # 96 | ### 方式1 97 | #h = Net::HTTP.new("mimvp.com",80) # 必须为 mimvp.com，不可以为 http://mimvp.com 98 | #resp,data = h.get("/") 99 | # 100 | #puts data # 空 101 | #puts resp # # 102 | #puts resp.body # 网页正文 103 | # 104 | # 105 | ### 方式2 106 | #resp = Net::HTTP.get_response(URI("http://mimvp.com")) # 必须为 http://mimvp.com，不可以为 mimvp.com 107 | #puts resp # # 108 | #puts resp.body # 网页正文 109 | # 110 | # 111 | ### 方式3 运行时需传入参数： ruby mimvp-proxy-ruby.rb mimvp.com 112 | #require 'uri' 113 | #require 'timeout' 114 | #require 'net/http' 115 | # 116 | #$resp = $data = nil 117 | # 118 | #begin 119 | # timeout(5) { 120 | # $resp = Net::HTTP.new(ARGV[0],80).get("/") # 方式1 121 | # $resp = Net::HTTP.get_response(URI("http://"+ARGV[0]+"/")) # 方式2 122 | # } 123 | #rescue => e 124 | # puts e.inspect 125 | # exit 126 | #end 127 | #puts $resp.body 128 | # 129 | ## 运行命令： ruby mimvp-proxy-ruby.rb mimvp.com 130 | 131 | 132 | -------------------------------------------------------------------------------- /SeleniumJava/MimvpProxy_FirefoxDriver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Selenum FirefoxDriver 支持 http、socks5 3 | * 4 | * 米扑代理示例： 5 | * http://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * http://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2015-11-09 12 | */ 13 | 14 | 15 | package com.mimvp; 16 | 17 | import org.openqa.selenium.firefox.FirefoxDriver; 18 | import org.openqa.selenium.firefox.FirefoxProfile; 19 | 20 | public class MimvpProxy_FirefoxDriver { 21 | 22 | final static String proxyUri = "103.35.171.77:8080"; // 代理服务器（HTTP, HTTPS） 23 | final static String proxySocks = "50.63.167.102:18628"; // 代理服务器（Socks5） 24 | final static String mimvpUrl = "http://proxy.mimvp.com/exist.php"; // 爬取网址 25 | 26 | // final static String firefoxBin = "D:/Program Files/Mozilla Firefox/firefox.exe"; // Windows 27 | // final static String firefoxBin = "/usr/bin/firefox-bin"; // Linux 28 | final static String firefoxBin = "/Applications/Firefox.app/Contents/MacOS/firefox-bin"; // Mac 29 | 30 | public static void main(String[] args) { 31 | getNoProxy(); 32 | getHttpProxy(); 33 | getSocksProxy(); 34 | } 35 | 36 | // 不用代理爬取网页 37 | public static void getNoProxy() { 38 | // System.setProperty("webdriver.firefox.bin", firefoxBin); // 不设置也可以 39 | 40 | FirefoxProfile profile = new FirefoxProfile(); 41 | 42 | // 启动Firefox，抓取网页后，再关闭Firefox 43 | FirefoxDriver driver = new FirefoxDriver(profile); 44 | driver.get(mimvpUrl); 45 | String title = driver.getTitle(); 46 | System.out.println(title); // 检测收录 - 米扑代理 47 | driver.quit(); 48 | driver.close(); 49 | } 50 | 51 | // HTTP和HTTPS代理爬取网页 52 | public static void getHttpProxy() { 53 | FirefoxProfile profile = new FirefoxProfile(); 54 | 55 | // 设置代理 56 | profile.setPreference("network.proxy.type", 1); // 0 - 不用代理； 1 - 使用代理 57 | profile.setPreference("network.proxy.http", proxyUri.split(":")[0]); 58 | profile.setPreference("network.proxy.http_port", Integer.parseInt(proxyUri.split(":")[1])); 59 | profile.setPreference("network.proxy.ssl", proxyUri.split(":")[0]); 60 | profile.setPreference("network.proxy.ssl_port", Integer.parseInt(proxyUri.split(":")[1])); 61 | profile.setPreference("network.proxy.no_proxies_on", "localhost"); 62 | 63 | // // 代理设置用户名和密码，无密码的代理无需配置 64 | // profile.setPreference("username", "mimvp-user"); 65 | // profile.setPreference("password", "mimvp-pwd"); 66 | 67 | FirefoxDriver driver = new FirefoxDriver(profile); 68 | driver.get(mimvpUrl); 69 | String title = driver.getTitle(); 70 | System.out.println(title); // 检测收录 - 米扑代理 71 | driver.close(); 72 | } 73 | 74 | // Socks5代理爬取网页 75 | public static void getSocksProxy() { 76 | FirefoxProfile profile = new FirefoxProfile(); 77 | 78 | // 设置代理 79 | profile.setPreference("network.proxy.type", 1); // 0 - 不用代理； 1 - 使用代理 80 | profile.setPreference("network.proxy.socks", proxySocks.split(":")[0]); 81 | profile.setPreference("network.proxy.socks_port", Integer.parseInt(proxySocks.split(":")[1])); 82 | profile.setPreference("network.proxy.no_proxies_on", "localhost"); 83 | 84 | FirefoxDriver driver = new FirefoxDriver(profile); 85 | driver.get(mimvpUrl); 86 | String title = driver.getTitle(); 87 | System.out.println(title); // 检测收录 - 米扑代理 88 | driver.close(); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /SeleniumJava/MimvpProxy_HtmlUnitDriver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Selenum HtmlUnitDriver 支持 http、socks5 3 | * 4 | * 米扑代理示例： 5 | * http://proxy.mimvp.com/demo2.php 6 | * 7 | * 米扑代理购买： 8 | * http://proxy.mimvp.com 9 | * 10 | * mimvp.com 11 | * 2015-11-09 12 | */ 13 | 14 | 15 | package com.mimvp; 16 | 17 | import org.openqa.selenium.By; 18 | import org.openqa.selenium.Keys; 19 | import org.openqa.selenium.Platform; 20 | import org.openqa.selenium.Proxy; 21 | import org.openqa.selenium.WebDriver; 22 | import org.openqa.selenium.htmlunit.HtmlUnitDriver; 23 | import org.openqa.selenium.interactions.Actions; 24 | import org.openqa.selenium.remote.CapabilityType; 25 | import org.openqa.selenium.remote.DesiredCapabilities; 26 | 27 | import com.gargoylesoftware.htmlunit.DefaultCredentialsProvider; 28 | import com.gargoylesoftware.htmlunit.WebClient; 29 | 30 | public class MimvpProxy_HtmlUnitDriver { 31 | final static String proxyUri = "183.222.102.98:8080"; // 代理服务器（HTTP） 32 | final static String proxySocks = "103.14.27.174:1080"; // 代理服务器（Socks5） 33 | final static String mimvpUrl = "http://proxy.mimvp.com/exist.php"; // 爬取网址 34 | 35 | public static void main(String[] args) { 36 | getNoProxy(); 37 | getHttpProxy(); 38 | getSocksProxy(); 39 | getAuthProxy(); 40 | getBaiduSearch("米扑科技"); 41 | } 42 | 43 | // 不用代理爬取网页 44 | public static void getNoProxy() { 45 | HtmlUnitDriver driver = new HtmlUnitDriver(true); // enable javascript 46 | driver.setJavascriptEnabled(true); 47 | driver.get(mimvpUrl); 48 | String title = driver.getTitle(); 49 | System.out.println(title); // 检测收录 - 米扑代理 50 | } 51 | 52 | 53 | // HTTP代理爬取网页 54 | public static void getHttpProxy() { 55 | HtmlUnitDriver driver = new HtmlUnitDriver(true); // enable javascript 56 | 57 | // 方法1 58 | driver.setProxy(proxyUri.split(":")[0], Integer.parseInt(proxyUri.split(":")[1])); // proxyUri = "183.222.102.98:8080" 59 | 60 | // 方式2 61 | driver.setHTTPProxy(proxyUri.split(":")[0], Integer.parseInt(proxyUri.split(":")[1]), null); // proxyUri = "183.222.102.98:8080" 62 | 63 | // 方法3 64 | Proxy proxy = new Proxy(); 65 | proxy.setHttpProxy(proxyUri); // 设置代理服务器地址, proxyUri = "183.222.102.98:8080" 66 | driver.setProxySettings(proxy); 67 | 68 | driver.get(mimvpUrl); 69 | 70 | String html = driver.getPageSource(); 71 | System.out.println(html); 72 | String title = driver.getTitle(); 73 | System.out.println(title); // 检测收录 - 米扑代理 74 | } 75 | 76 | 77 | // Socks5代理爬取网页 78 | public static void getSocksProxy() { 79 | HtmlUnitDriver driver = new HtmlUnitDriver(true); // enable javascript 80 | 81 | // 方式1 82 | driver.setSocksProxy(proxySocks.split(":")[0], Integer.parseInt(proxySocks.split(":")[1])); // proxySocks = "183.239.240.138:1080" 83 | 84 | // 方式2 85 | driver.setSocksProxy(proxySocks.split(":")[0], Integer.parseInt(proxySocks.split(":")[1]), null); // proxySocks = "183.239.240.138:1080" 86 | 87 | driver.get(mimvpUrl); 88 | 89 | String html = driver.getPageSource(); 90 | System.out.println(html); 91 | String title = driver.getTitle(); 92 | System.out.println(title); // 检测收录 - 米扑代理 93 | } 94 | 95 | 96 | // 代理需要用户名和密码 97 | public static void getAuthProxy() { 98 | HtmlUnitDriver driver = null; 99 | 100 | final String proxyUser = "mimvp-user"; 101 | final String proxyPass = "mimvp-pwd"; 102 | 103 | Proxy proxy = new Proxy(); 104 | proxy.setHttpProxy(proxyUri); 105 | 106 | // 设置代理的用户名和密码 107 | DesiredCapabilities capabilities = DesiredCapabilities.htmlUnit(); 108 | capabilities.setCapability(CapabilityType.PROXY, proxy); 109 | capabilities.setJavascriptEnabled(true); 110 | capabilities.setPlatform(Platform.WIN8_1); 111 | driver = new HtmlUnitDriver(capabilities) { 112 | @Override 113 | protected WebClient modifyWebClient(WebClient client) { 114 | DefaultCredentialsProvider creds = new DefaultCredentialsProvider(); 115 | creds.addCredentials(proxyUser, proxyPass); 116 | client.setCredentialsProvider(creds); 117 | return client; 118 | } 119 | }; 120 | driver.setJavascriptEnabled(true); // enable javascript 121 | driver.get(mimvpUrl); 122 | String title = driver.getTitle(); 123 | System.out.println(title); // 检测收录 - 米扑代理 124 | } 125 | 126 | 127 | // 进行百度搜索 128 | public static void getBaiduSearch(String keyword) { 129 | final String url = "http://www.baidu.com"; 130 | WebDriver driver = new HtmlUnitDriver(false); 131 | driver.get(url); 132 | driver.findElement(By.id("kw")).sendKeys(keyword); 133 | Actions action = new Actions(driver); 134 | action.sendKeys(Keys.ENTER).perform(); 135 | String html = driver.getPageSource(); 136 | System.out.println(html); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /SeleniumPython/ghostdriver.log: -------------------------------------------------------------------------------- 1 | [INFO - 2017-07-28T16:56:39.352Z] GhostDriver - Main - running on port 51562 2 | [INFO - 2017-07-28T16:56:39.384Z] Session [b9563940-73b5-11e7-a0e9-e37a1c35d385] - page.settings - {"XSSAuditingEnabled":false,"javascriptCanCloseWindows":true,"javascriptCanOpenWindows":true,"javascriptEnabled":true,"loadImages":true,"localToRemoteUrlAccessEnabled":false,"userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1","webSecurityEnabled":true} 3 | [INFO - 2017-07-28T16:56:39.384Z] Session [b9563940-73b5-11e7-a0e9-e37a1c35d385] - page.customHeaders: - {} 4 | [INFO - 2017-07-28T16:56:39.384Z] Session [b9563940-73b5-11e7-a0e9-e37a1c35d385] - Session.negotiatedCapabilities - {"browserName":"phantomjs","version":"2.1.1","driverName":"ghostdriver","driverVersion":"1.2.0","platform":"mac-unknown-64bit","javascriptEnabled":true,"takesScreenshot":true,"handlesAlerts":false,"databaseEnabled":false,"locationContextEnabled":false,"applicationCacheEnabled":false,"browserConnectionEnabled":false,"cssSelectorsEnabled":true,"webStorageEnabled":false,"rotatable":false,"acceptSslCerts":false,"nativeEvents":true,"proxy":{"proxyType":"direct"}} 5 | [INFO - 2017-07-28T16:56:39.384Z] SessionManagerReqHand - _postNewSessionCommand - New Session Created: b9563940-73b5-11e7-a0e9-e37a1c35d385 6 | -------------------------------------------------------------------------------- /SeleniumPython/mimvp-proxy-phantomjs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Selenium PhantomJS 支持 http、socks5 5 | # 6 | # 米扑代理示例： 7 | # http://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # http://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2017-01-08 14 | 15 | 16 | from selenium import webdriver 17 | 18 | 19 | proxy_http = "http://138.68.165.154:3128" 20 | proxy_socks5 = "socks5://209.151.135.126:11747" 21 | 22 | mimvp_url = "http://proxy.mimvp.com/exist.php" 23 | mimvp_url2 = "https://proxy.mimvp.com/exist.php" 24 | mimvp_url3 = "https://apps.bdimg.com/libs/jquery-i18n/1.1.1/jquery.i18n.min.js" 25 | 26 | 27 | 28 | # urllib2 支持 http, https 29 | def test_http(proxy, mimvp_url): 30 | 31 | proxy_type = proxy.split("://")[0] 32 | proxy_ip = proxy.split("://")[1].split(":")[0] 33 | proxy_port = int(proxy.split("://")[1].split(":")[1]) 34 | 35 | # 使用PhantomJS命令，因PhantomJS仅支持http、socks5，所以Selenium Webdriver也只支持http、socks5 36 | service_args = [ 37 | "--proxy-type=%(http)s" % {"http" : proxy_type}, 38 | "--proxy=%(host)s:%(port)s" % { 39 | "host" : proxy_ip, 40 | "port" : proxy_port, 41 | }, 42 | # "--proxy-auth=%(user)s:%(pass)s" % { 43 | # "user" : 'mimvp-user', 44 | # "pass" : 'mimvp-pass', 45 | # }, 46 | ] 47 | 48 | phantomjs_path = r"/opt/phantomjs-2.1.1/bin/phantomjs" 49 | 50 | driver = webdriver.PhantomJS(executable_path=phantomjs_path, service_args=service_args) 51 | driver.get(mimvp_url) 52 | 53 | print driver.title 54 | print driver.page_source.encode("utf-8") 55 | 56 | driver.quit() 57 | 58 | 59 | 60 | if __name__ == "__main__": 61 | # http, socks5 62 | test_http(proxy_http, mimvp_url) 63 | test_http(proxy_socks5, mimvp_url) 64 | -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/Chrome-proxy-helper/background.js: -------------------------------------------------------------------------------- 1 | var config = { 2 | mode: "fixed_servers", 3 | rules: { 4 | singleProxy: { 5 | scheme: "http", 6 | host: "mimvp_proxy_host", 7 | port: parseInt(mimvp_proxy_port) 8 | }, 9 | bypassList: ["mimvp.com"] 10 | } 11 | }; 12 | 13 | chrome.proxy.settings.set({value: config, scope: "regular"}, function() {}); 14 | 15 | function callbackFn(details) { 16 | return { 17 | authCredentials: { 18 | username: "mimvp_username", 19 | password: "mimvp_password" 20 | } 21 | }; 22 | } 23 | 24 | chrome.webRequest.onAuthRequired.addListener( 25 | callbackFn, 26 | {urls: [""]}, 27 | ['blocking'] 28 | ); -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/Chrome-proxy-helper/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0.0", 3 | "manifest_version": 2, 4 | "name": "Chrome Proxy", 5 | "permissions": [ 6 | "proxy", 7 | "tabs", 8 | "unlimitedStorage", 9 | "storage", 10 | "", 11 | "webRequest", 12 | "webRequestBlocking" 13 | ], 14 | "background": { 15 | "scripts": ["background.js"] 16 | }, 17 | "minimum_chrome_version":"22.0.0" 18 | } -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/chrome-proxy-extensions/mimvp-user_mimvp-pass@140.143.62.84_19480.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/SeleniumPython/python_selenium_chrome/chrome-proxy-extensions/mimvp-user_mimvp-pass@140.143.62.84_19480.zip -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/chrome-proxy-extensions/mimvp-user_mimvp-pass@39.106.158.153_14051.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/SeleniumPython/python_selenium_chrome/chrome-proxy-extensions/mimvp-user_mimvp-pass@39.106.158.153_14051.zip -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/chrome-proxy-extensions/mimvp-user_mimvp-pass@47.104.234.80_53848.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/SeleniumPython/python_selenium_chrome/chrome-proxy-extensions/mimvp-user_mimvp-pass@47.104.234.80_53848.zip -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/proxy.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/SeleniumPython/python_selenium_chrome/proxy.zip -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/readme.txt: -------------------------------------------------------------------------------- 1 | 2 | chromedriver 下载地址： 3 | 4 | http://chromedriver.storage.googleapis.com/index.html 5 | 6 | [DIR] chromedriver_linux64.zip 2018-04-21 22:31:09 3.60MB 7 | 8 | [DIR] chromedriver_mac64.zip 2018-04-21 22:31:10 5.32MB 9 | 10 | [DIR] chromedriver_win32.zip 2018-04-21 22:31:11 3.22MB 11 | 12 | [DIR] notes.txt 2018-04-18 15:51:23 0.01MB 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_chrome/test_selenium_chrome.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Selenium + Chrome 支持 http、https (无密和加密两种方式)，不支持socks5 5 | # 6 | # 米扑代理示例： 7 | # https://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # https://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2017-01-08 14 | 15 | # Python + Selenium + Chrome 代理示例，详见米扑博客： 16 | # https://blog.mimvp.com/article/25076.html 17 | # 18 | # 本示例由米扑代理原创，测试代理来自于米扑代理 19 | # 密码授权和白名单ip设置，请见米扑代理 - 会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 20 | 21 | 22 | from selenium import webdriver 23 | from selenium.webdriver.common.proxy import * 24 | from selenium.webdriver.chrome.options import Options 25 | from selenium.webdriver.common.desired_capabilities import DesiredCapabilities 26 | from pyvirtualdisplay import Display 27 | # from xvfbwrapper import Xvfb 28 | 29 | import bs4, os, re, time, zipfile 30 | from base64 import b64encode 31 | 32 | import sys 33 | from posix import unlink 34 | reload(sys) 35 | sys.setdefaultencoding('utf8') 36 | 37 | 38 | ## webdriver + chrome （不使用代理，爬取网页） 39 | def spider_url_chrome(url): 40 | browser = None 41 | display = None 42 | try: 43 | display = Display(visible=0, size=(800, 600)) 44 | display.start() 45 | chromedriver = '/usr/local/bin/chromedriver' 46 | browser = webdriver.Chrome(executable_path=chromedriver) # 打开 Chrome 浏览器 47 | browser.get(url) 48 | content = browser.page_source 49 | print("content: " + str(content)) 50 | finally: 51 | if browser: browser.quit() 52 | if display: display.stop() 53 | 54 | 55 | ## webdriver + chrome + proxy + whiteip （无密码，或白名单ip授权） 56 | ## 米扑代理：https://proxy.mimvp.com 57 | def spider_url_chrome_by_whiteip(url): 58 | browser = None 59 | display = None 60 | 61 | ## 白名单ip，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 62 | mimvp_proxy = { 63 | 'ip' : '140.143.62.84', # ip 64 | 'port_https' : 62288, # http, https 65 | 'port_socks' : 62287, # socks5 66 | 'username' : 'mimvp-user', 67 | 'password' : 'mimvp-pass' 68 | } 69 | 70 | try: 71 | display = Display(visible=0, size=(800, 600)) 72 | display.start() 73 | 74 | chrome_options = Options() # ok 75 | chrome_options = webdriver.ChromeOptions() # ok 76 | proxy_https_argument = '--proxy-server=http://{ip}:{port}'.format(ip=mimvp_proxy['ip'], port=mimvp_proxy['port_https']) # http, https (无密码，或白名单ip授权，成功) 77 | chrome_options.add_argument(proxy_https_argument) 78 | # proxy_socks_argument = '--proxy-server=socks5://{ip}:{port}'.format(ip=mimvp_proxy['ip'], port=mimvp_proxy['port_socks']) # socks5 (无密码，或白名单ip授权，失败) 79 | # chrome_options.add_argument(proxy_socks_argument) 80 | 81 | chromedriver = '/usr/local/bin/chromedriver' 82 | browser = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_options) # 打开 Chrome 浏览器 83 | browser.get(url) 84 | content = browser.page_source 85 | print("content: " + str(content)) 86 | finally: 87 | if browser: browser.quit() 88 | if display: display.stop() 89 | 90 | 91 | 92 | ## webdriver + chrome + proxy + https （https密码授权） 93 | ## 米扑代理：https://proxy.mimvp.com 94 | def spider_url_chrome_by_https(url): 95 | browser = None 96 | display = None 97 | try: 98 | display = Display(visible=0, size=(800, 600)) 99 | display.start() 100 | 101 | chrome_options = Options() # ok 102 | chrome_options = webdriver.ChromeOptions() # ok 103 | chrome_options.add_extension("proxy.zip") ## 手动打zip包，包含 background.js 和 manifest.json 两个文件 104 | 105 | chromedriver = '/usr/local/bin/chromedriver' 106 | browser = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_options) # 打开 Chrome 浏览器 107 | browser.get(url) 108 | content = browser.page_source 109 | print("content: " + str(content)) 110 | finally: 111 | if browser: browser.quit() 112 | if display: display.stop() 113 | 114 | 115 | 116 | ## 自动打zip包，包含 background.js 和 manifest.json 两个文件 117 | def get_chrome_proxy_extension(proxy): 118 | """获取一个Chrome代理扩展,里面配置有指定的代理(带用户名密码认证) 119 | proxy - 指定的代理,格式: username:password@ip:port 120 | """ 121 | 122 | # Chrome代理插件的参考模板 https://github.com/RobinDev/Selenium-Chrome-HTTP-Private-Proxy 123 | CHROME_PROXY_HELPER_DIR = 'Chrome-proxy-helper' # 自定义目录名，放在代理项目的当前同一级目录 124 | 125 | # 存储自定义Chrome代理扩展文件的目录，一般为当前同一级目录 126 | # 生成的zip路径为：chrome-proxy-extensions/mimvp-user_mimvp-pass@140.143.62.84_19480.zip 127 | CUSTOM_CHROME_PROXY_EXTENSIONS_DIR = 'chrome-proxy-extensions' 128 | 129 | m = re.compile('([^:]+):([^\@]+)\@([\d\.]+):(\d+)').search(proxy) 130 | if m: 131 | # 提取代理的各项参数 132 | username = m.groups()[0] 133 | password = m.groups()[1] 134 | ip = m.groups()[2] 135 | port = m.groups()[3] 136 | # 创建一个定制Chrome代理扩展(zip文件) 137 | if not os.path.exists(CUSTOM_CHROME_PROXY_EXTENSIONS_DIR): 138 | os.mkdir(CUSTOM_CHROME_PROXY_EXTENSIONS_DIR) 139 | extension_file_path = os.path.join(CUSTOM_CHROME_PROXY_EXTENSIONS_DIR, '{}.zip'.format(proxy.replace(':', '_'))) 140 | 141 | # 扩展文件不存在，则创建配置文件，并写入zip文件 142 | if not os.path.exists(extension_file_path): 143 | zf = zipfile.ZipFile(extension_file_path, mode='w') 144 | zf.write(os.path.join(CHROME_PROXY_HELPER_DIR, 'manifest.json'), 'manifest.json') 145 | # 替换模板中的代理参数 146 | background_content = open(os.path.join(CHROME_PROXY_HELPER_DIR, 'background.js')).read() 147 | background_content = background_content.replace('mimvp_proxy_host', ip) 148 | background_content = background_content.replace('mimvp_proxy_port', port) 149 | background_content = background_content.replace('mimvp_username', username) 150 | background_content = background_content.replace('mimvp_password', password) 151 | zf.writestr('background.js', background_content) 152 | zf.close() 153 | return extension_file_path 154 | else: 155 | raise Exception('Invalid proxy format. Should be username:password@ip:port') 156 | 157 | 158 | ## webdriver + chrome + proxy + https （https密码授权，自动打包zip） 159 | ## 米扑代理：https://proxy.mimvp.com 160 | def spider_url_chrome_by_https2(url): 161 | browser = None 162 | display = None 163 | try: 164 | display = Display(visible=0, size=(800, 600)) 165 | display.start() 166 | 167 | proxy = 'mimvp-user:mimvp-pass@118.24.232.74:53681' 168 | proxy_zip = get_chrome_proxy_extension(proxy) # 打包代理zip文件 169 | 170 | chrome_options = Options() # ok 171 | chrome_options = webdriver.ChromeOptions() # ok 172 | chrome_options.add_extension(proxy_zip) 173 | 174 | chromedriver = '/usr/local/bin/chromedriver' 175 | browser = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_options) # 打开 Chrome 浏览器 176 | browser.get(url) 177 | content = browser.page_source 178 | print("content: " + str(content)) 179 | 180 | unlink(proxy_zip) # 使用代理后，删除代理zip文件 181 | finally: 182 | if browser: browser.quit() 183 | if display: display.stop() 184 | 185 | 186 | 187 | ## webdriver + chrome + proxy + socks （socks密码授权） 188 | ## 米扑代理：https://proxy.mimvp.com 189 | def spider_url_chrome_by_socks(url): 190 | browser = None 191 | display = None 192 | 193 | ## 白名单ip，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 194 | mimvp_proxy = { 195 | 'ip' : '140.143.62.84', # ip 196 | 'port_https' : 62288, # http, https 197 | 'port_socks' : 62289, # socks5 198 | 'username' : 'mimvp-user', 199 | 'password' : 'mimvp-pass' 200 | } 201 | 202 | try: 203 | display = Display(visible=0, size=(800, 600)) 204 | display.start() 205 | 206 | capabilities = dict(DesiredCapabilities.CHROME) 207 | capabilities['proxy'] = { 208 | 'proxyType' : 'MANUAL', 209 | # 'httpProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']), 210 | # 'sslProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']), 211 | 'socksProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_socks']), 212 | 'ftpProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']), 213 | 'noProxy' : 'localhost,127.0.0.1', 214 | 'class' : "org.openqa.selenium.Proxy", 215 | 'autodetect' : False 216 | } 217 | 218 | capabilities['proxy']['socksUsername'] = mimvp_proxy['username'] 219 | capabilities['proxy']['socksPassword'] = mimvp_proxy['password'] 220 | 221 | chromedriver = '/usr/local/bin/chromedriver' 222 | browser = webdriver.Chrome(chromedriver, desired_capabilities=capabilities) 223 | browser.get(url) 224 | content = browser.page_source 225 | print("content: " + str(content)) 226 | finally: 227 | if browser: browser.quit() 228 | if display: display.stop() 229 | 230 | 231 | 232 | if __name__ == '__main__': 233 | url = 'https://mimvp.com/' 234 | url = 'https://proxy.mimvp.com/ip.php' 235 | 236 | # 不使用代理，爬取网页 237 | spider_url_chrome(url) 238 | 239 | # 代理无密码，或设置白名单ip，成功 240 | spider_url_chrome_by_whiteip(url) 241 | 242 | # http, https 密码授权，成功 243 | spider_url_chrome_by_https(url) 244 | spider_url_chrome_by_https2(url) 245 | 246 | # socks5 密码授权，失败 （仍然是本机ip请求的，不是代理ip请求） 247 | spider_url_chrome_by_socks(url) 248 | 249 | 250 | -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_firefox/close_proxy_authentication-1.1.xpi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/SeleniumPython/python_selenium_firefox/close_proxy_authentication-1.1.xpi -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_firefox/modify_headers-0.7.1.1-fx.xpi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/SeleniumPython/python_selenium_firefox/modify_headers-0.7.1.1-fx.xpi -------------------------------------------------------------------------------- /SeleniumPython/python_selenium_firefox/test_selenium_firefox.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Selenium + Firefox 支持 http、https (无密和加密两种方式)，不支持socks5 5 | # 6 | # 米扑代理示例： 7 | # https://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # https://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2017-01-08 14 | 15 | # Python + Selenium + Firefox 设置密码时，需要使用到两个插件： 16 | # 插件1： modify_headers-0.7.1.1-fx.xpi 17 | # 下载地址：https://github.com/mimvp/mimvp-proxy-demo 18 | # 19 | # 方式2： close_proxy_authentication-1.1.xpi 20 | # 下载地址：https://github.com/mimvp/mimvp-proxy-demo 21 | # 22 | # Python + Selenium + Firefox 代理示例，详见米扑博客： 23 | # https://blog.mimvp.com/article/25055.html 24 | # 25 | # 本示例由米扑代理原创，测试代理来自于米扑代理 26 | # 密码授权和白名单ip设置，请见米扑代理 - 会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 27 | 28 | 29 | from selenium import webdriver 30 | from selenium.webdriver.firefox.firefox_binary import FirefoxBinary 31 | from selenium.webdriver.common.proxy import * 32 | from pyvirtualdisplay import Display 33 | # from xvfbwrapper import Xvfb 34 | 35 | import bs4, os 36 | from base64 import b64encode 37 | 38 | import sys 39 | reload(sys) 40 | sys.setdefaultencoding('utf8') 41 | 42 | 43 | ## webdriver + firefox （不使用代理，爬取网页） 44 | def spider_url_firefox(url): 45 | browser = None 46 | display = None 47 | try: 48 | display = Display(visible=0, size=(800, 600)) 49 | display.start() 50 | browser = webdriver.Firefox() # 打开 FireFox 浏览器 51 | browser.get(url) 52 | content = browser.page_source 53 | print("content: " + str(content)) 54 | finally: 55 | if browser: browser.quit() 56 | if display: display.stop() 57 | 58 | 59 | ## webdriver + firefox + proxy + whiteip （无密码，或白名单ip授权） 60 | ## 米扑代理：https://proxy.mimvp.com 61 | def spider_url_firefox_by_whiteip(url): 62 | browser = None 63 | display = None 64 | 65 | ## 白名单ip，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 66 | mimvp_proxy = { 67 | 'ip' : '140.143.62.84', # ip 68 | 'port_https' : 19480, # http, https 69 | 'port_socks' : 19481, # socks5 70 | 'username' : 'mimvp-user', 71 | 'password' : 'mimvp-pass' 72 | } 73 | 74 | try: 75 | display = Display(visible=0, size=(800, 600)) 76 | display.start() 77 | 78 | profile = webdriver.FirefoxProfile() 79 | 80 | # add proxy 81 | profile.set_preference('network.proxy.type', 1) # ProxyType.MANUAL = 1 82 | if url.startswith("http://"): 83 | profile.set_preference('network.proxy.http', mimvp_proxy['ip']) 84 | profile.set_preference('network.proxy.http_port', mimvp_proxy['port_https']) # 访问http网站 85 | elif url.startswith("https://"): 86 | profile.set_preference('network.proxy.ssl', mimvp_proxy['ip']) 87 | profile.set_preference('network.proxy.ssl_port', mimvp_proxy['port_https']) # 访问https网站 88 | else: 89 | profile.set_preference('network.proxy.socks', mimvp_proxy['ip']) 90 | profile.set_preference('network.proxy.socks_port', mimvp_proxy['port_socks']) 91 | profile.set_preference('network.proxy.ftp', mimvp_proxy['ip']) 92 | profile.set_preference('network.proxy.ftp_port', mimvp_proxy['port_https']) 93 | profile.set_preference('network.proxy.no_proxies_on', 'localhost,127.0.0.1') 94 | 95 | ## 不存在此用法，不能这么设置用户名密码 （舍弃） 96 | # profile.set_preference("network.proxy.username", 'mimvp-user') 97 | # profile.set_preference("network.proxy.password", 'mimvp-pass') 98 | 99 | profile.update_preferences() 100 | 101 | browser = webdriver.Firefox(profile) # 打开 FireFox 浏览器 102 | browser.get(url) 103 | content = browser.page_source 104 | print("content: " + str(content)) 105 | finally: 106 | if browser: browser.quit() 107 | if display: display.stop() 108 | 109 | 110 | ## webdriver + firefox + proxy + https （https密码授权） 111 | ## 米扑代理：https://proxy.mimvp.com 112 | def spider_url_firefox_by_https(url): 113 | browser = None 114 | display = None 115 | 116 | ## 授权密码，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 117 | mimvp_proxy = { 118 | 'ip' : '140.143.62.84', # ip 119 | 'port_https' : 19480, # http, https 120 | 'port_socks' : 19481, # socks5 121 | 'username' : 'mimvp-user', 122 | 'password' : 'mimvp-pass' 123 | } 124 | 125 | try: 126 | display = Display(visible=0, size=(800, 600)) 127 | display.start() 128 | 129 | profile = webdriver.FirefoxProfile() 130 | 131 | # add new header 132 | profile.add_extension("modify_headers-0.7.1.1-fx.xpi") 133 | profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx") 134 | profile.set_preference("modifyheaders.config.active", True) 135 | profile.set_preference("modifyheaders.headers.count", 1) 136 | profile.set_preference("modifyheaders.headers.action0", "Add") 137 | profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip") 138 | profile.set_preference("modifyheaders.headers.value0", "yes") 139 | profile.set_preference("modifyheaders.headers.enabled0", True) 140 | 141 | # add proxy 142 | profile.set_preference('network.proxy.type', 1) # ProxyType.MANUAL = 1 143 | if url.startswith("http://"): 144 | profile.set_preference('network.proxy.http', mimvp_proxy['ip']) 145 | profile.set_preference('network.proxy.http_port', mimvp_proxy['port_https']) # 访问http网站 146 | elif url.startswith("https://"): 147 | profile.set_preference('network.proxy.ssl', mimvp_proxy['ip']) 148 | profile.set_preference('network.proxy.ssl_port', mimvp_proxy['port_https']) # 访问https网站 149 | 150 | # Proxy auto login （自动填写密码，进行代理授权） 151 | profile.add_extension('close_proxy_authentication-1.1.xpi') 152 | credentials = '{username}:{password}'.format(username=mimvp_proxy['username'], password=mimvp_proxy['password']) # auth 153 | credentials = b64encode(credentials.encode('ascii')).decode('utf-8') 154 | profile.set_preference('extensions.closeproxyauth.authtoken', credentials) 155 | 156 | profile.update_preferences() 157 | 158 | browser = webdriver.Firefox(profile) # 打开 FireFox 浏览器 159 | browser.get(url) 160 | content = browser.page_source 161 | print("content: " + str(content)) 162 | finally: 163 | if browser: browser.quit() 164 | if display: display.stop() 165 | 166 | 167 | ## webdriver + firefox + proxy + socks （socks密码授权） 168 | ## 米扑代理：https://proxy.mimvp.com 169 | def spider_url_firefox_by_socks(url): 170 | browser = None 171 | display = None 172 | 173 | ## 授权密码，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip 174 | mimvp_proxy = { 175 | 'ip' : '140.143.62.84', # ip 176 | 'port_https' : 62288, # http, https 177 | 'port_socks' : 62287, # socks5 178 | 'username' : 'mimvp-user', 179 | 'password' : 'mimvp-pass' 180 | } 181 | 182 | proxy_config = Proxy({ 183 | 'proxyType' : ProxyType.MANUAL, # 1 184 | 'httpProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']), 185 | 'sslProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']), 186 | 'socksProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_socks']), 187 | 'ftpProxy' : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']), 188 | 'noProxy' : 'localhost,127.0.0.1', 189 | 'socksUsername' : mimvp_proxy['username'], 190 | 'socksPassword' : mimvp_proxy['password'], 191 | }) 192 | 193 | try: 194 | display = Display(visible=0, size=(800, 600)) 195 | display.start() 196 | 197 | profile = webdriver.FirefoxProfile() 198 | 199 | # add new header 200 | profile.add_extension("modify_headers-0.7.1.1-fx.xpi") 201 | profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx") 202 | profile.set_preference("modifyheaders.config.active", True) 203 | profile.set_preference("modifyheaders.headers.count", 1) 204 | profile.set_preference("modifyheaders.headers.action0", "Add") 205 | profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip") 206 | profile.set_preference("modifyheaders.headers.value0", "yes") 207 | profile.set_preference("modifyheaders.headers.enabled0", True) 208 | 209 | # auto save auth 210 | profile.set_preference("signon.autologin.proxy", 'true') 211 | profile.set_preference("network.websocket.enabled", 'false') 212 | profile.set_preference('network.proxy.share_proxy_settings', 'false') 213 | profile.set_preference('network.automatic-ntlm-auth.allow-proxies', 'false') 214 | profile.set_preference('network.auth.use-sspi', 'false') 215 | profile.update_preferences() 216 | 217 | browser = webdriver.Firefox(proxy=proxy_config, firefox_profile=profile) # 打开 FireFox 浏览器 218 | browser.get(url) 219 | content = browser.page_source 220 | print("content: " + str(content)) 221 | finally: 222 | if browser: browser.quit() 223 | if display: display.stop() 224 | 225 | 226 | if __name__ == '__main__': 227 | url = 'https://ip.cn' 228 | url = 'https://mimvp.com' 229 | url = 'https://proxy.mimvp.com/ip.php' 230 | 231 | # 不使用代理，爬取网页 232 | spider_url_firefox(url) 233 | 234 | # 代理无密码，或设置白名单ip，成功 235 | spider_url_firefox_by_whiteip(url) 236 | 237 | # http, https 密码授权，成功 238 | spider_url_firefox_by_https(url) 239 | 240 | # socks5 密码授权，失败 （仍然是本机ip请求的，不是代理ip请求） 241 | spider_url_firefox_by_socks(url) 242 | 243 | 244 | -------------------------------------------------------------------------------- /Shell/mimvp-proxy-shell.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # curl 支持 http、https、socks4、socks5 4 | # wget 支持 http、https 5 | # 6 | # 米扑代理示例： 7 | # https://proxy.mimvp.com/demo2.php 8 | # 9 | # 米扑代理购买： 10 | # https://proxy.mimvp.com 11 | # 12 | # mimvp.com 13 | # 2015-11-09 14 | 15 | #【米扑代理】：本示例，在CentOS、Ubuntu、MacOS等服务器上，均测试通过 16 | # 17 | # http代理格式 http_proxy=http://IP:Port 18 | # https代理格式 https_proxy=http://IP:Port 19 | 20 | 21 | 22 | ################### proxy no auth ################### 23 | 24 | # curl和wget，爬取http网页 25 | {'http': 'http://120.77.176.179:8888'} 26 | curl -m 30 --retry 3 -x http://120.77.176.179:8888 http://proxy.mimvp.com/test_proxy2.php # http_proxy 27 | wget -T 30 --tries 3 -e "http_proxy=http://120.77.176.179:8888" http://proxy.mimvp.com/test_proxy2.php # http_proxy 28 | 29 | # curl和wget，爬取https网页（注意：添加参数，不经过SSL安全验证） 30 | {'https': 'http://46.105.214.133:3128'} 31 | curl -m 30 --retry 3 -x http://46.105.214.133:3128 -k https://proxy.mimvp.com/test_proxy2.php # https_proxy 32 | wget -T 30 --tries 3 -e "https_proxy=http://46.105.214.133:3128" --no-check-certificate https://proxy.mimvp.com/test_proxy2.php # https_proxy 33 | 34 | 35 | # curl 支持socks 36 | # 其中，socks4和socks5两种协议的代理，都可以同时爬取http和https网页 37 | {'socks4': '101.255.17.145:1080'} 38 | curl -m 30 --retry 3 --socks4 101.255.17.145:1080 http://proxy.mimvp.com/test_proxy2.php 39 | curl -m 30 --retry 3 --socks4 101.255.17.145:1080 https://proxy.mimvp.com/test_proxy2.php 40 | 41 | {'socks5': '82.164.233.227:45454'} 42 | curl -m 30 --retry 3 --socks5 82.164.233.227:45454 http://proxy.mimvp.com/test_proxy2.php 43 | curl -m 30 --retry 3 --socks5 82.164.233.227:45454 https://proxy.mimvp.com/test_proxy2.php 44 | 45 | # wget 不支持socks 46 | 47 | 48 | 49 | 50 | ################### proxy auth（代理需要用户名和密码验证）################### 51 | 52 | # curl和wget，爬取http网页 53 | curl -m 30 --retry 3 -x http://username:password@210.159.166.225:5718 http://proxy.mimvp.com/test_proxy2.php # http 54 | curl -m 30 --retry 3 -x http://username:password@210.159.166.225:5718 https://proxy.mimvp.com/test_proxy2.php # https 55 | curl -m 30 --retry 3 -U username:password -x http://210.159.166.225:5718 http://proxy.mimvp.com/test_proxy2.php # http 56 | curl -m 30 --retry 3 -U username:password -x http://210.159.166.225:5718 https://proxy.mimvp.com/test_proxy2.php # https 57 | curl -m 30 --retry 3 --proxy-user username:password -x http://210.159.166.225:5718 http://proxy.mimvp.com/test_proxy2.php # http 58 | curl -m 30 --retry 3 --proxy-user username:password -x http://210.159.166.225:5718 https://proxy.mimvp.com/test_proxy2.php # https 59 | 60 | wget -T 30 --tries 3 -e "http_proxy=http://username:password@2.19.16.5:5718" http://proxy.mimvp.com/test_proxy2.php 61 | wget -T 30 --tries 3 -e "https_proxy=http://username:password@2.19.16.5:5718" https://proxy.mimvp.com/test_proxy2.php 62 | wget -T 30 --tries 3 --proxy-user=username --proxy-password=password -e "http_proxy=http://2.19.16.5:5718" http://proxy.mimvp.com/test_proxy2.php 63 | wget -T 30 --tries 3 --proxy-user=username --proxy-password=password -e "https_proxy=http://2.19.16.5:5718" https://proxy.mimvp.com/test_proxy2.php 64 | 65 | 66 | # curl 支持socks 67 | curl -m 30 --retry 3 -U username:password --socks5 21.59.126.22:57216 http://proxy.mimvp.com/test_proxy2.php # socks5 68 | curl -m 30 --retry 3 -U username:password --socks5 21.59.126.22:57216 https://proxy.mimvp.com/test_proxy2.php # socks5 69 | curl -m 30 --retry 3 --proxy-user username:password --socks5 21.59.126.22:57216 http://proxy.mimvp.com/test_proxy2.php # socks5 70 | curl -m 30 --retry 3 --proxy-user username:password --socks5 21.59.126.22:57216 https://proxy.mimvp.com/test_proxy2.php # socks5 71 | 72 | # wget 不支持socks 73 | 74 | 75 | 76 | 77 | ################### wget配置文件设置代理 ################### 78 | 79 | vim ~/.wgetrc 80 | 81 | http_proxy=http://120.77.176.179:8888:8080 82 | https_proxy=http://12.7.17.17:8888:8080 83 | use_proxy = on 84 | wait = 30 85 | 86 | # 配置文件设置后，立即生效，直接执行wget爬取命令即可 87 | wget -T 30 --tries 3 http://proxy.mimvp.com/test_proxy2.php 88 | wget -T 30 --tries 3 https://proxy.mimvp.com/test_proxy2.php 89 | 90 | 91 | 92 | 93 | ################### 设置临时局部代理 ################### 94 | 95 | # proxy no auth 96 | export http_proxy=http://120.77.176.179:8888:8080 97 | export https_proxy=http://12.7.17.17:8888:8080 98 | 99 | # proxy auth（代理需要用户名和密码验证） 100 | export http_proxy=http://username:password@120.77.176.179:8888:8080 101 | export https_proxy=http://username:password@12.7.17.17:8888:8080 102 | 103 | # 直接爬取网页 104 | curl -m 30 --retry 3 http://proxy.mimvp.com/test_proxy2.php # http_proxy 105 | curl -m 30 --retry 3 https://proxy.mimvp.com/test_proxy2.php # https_proxy 106 | wget -T 30 --tries 3 http://proxy.mimvp.com/test_proxy2.php # http_proxy 107 | wget -T 30 --tries 3 https://proxy.mimvp.com/test_proxy2.php # https_proxy 108 | 109 | # 取消设置 110 | unset http_proxy 111 | unset https_proxy 112 | 113 | 114 | 115 | 116 | ################### 设置系统全局代理 ################### 117 | 118 | # 修改 /etc/profile，保存并重启服务器 119 | sudo vim /etc/profile # 所有人有效 120 | 或 121 | sudo vim ~/.bashrc # 所有人有效 122 | 或 123 | vim ~/.bash_profile # 个人有效 124 | 125 | 126 | ## 在文件末尾，添加如下内容 127 | # proxy no auth 128 | export http_proxy=http://120.77.176.179:8888:8080 129 | export https_proxy=http://12.7.17.17:8888:8080 130 | 131 | # proxy auth（代理需要用户名和密码验证） 132 | export http_proxy=http://username:password@120.77.176.179:8888:8080 133 | export https_proxy=http://username:password@12.7.17.17:8888:8080 134 | 135 | 136 | ## 执行source命令，使配置文件生效（临时生效） 137 | source /etc/profile 138 | 或 139 | source ~/.bashrc 140 | 或 141 | source ~/.bash_profile 142 | 143 | 144 | ## 若需要机器永久生效，则需要重启服务器 145 | sudo reboot 146 | -------------------------------------------------------------------------------- /User-Agent.txt: -------------------------------------------------------------------------------- 1 | /** 2 | 米扑代理示例： 3 | https://proxy.mimvp.com/demo2.php 4 | 5 | 米扑代理购买： 6 | https://proxy.mimvp.com 7 | 8 | mimvp.com 9 | 2014-09-12 10 | */ 11 | 12 | Mozilla/5.0 13 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729) 14 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 15 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 16 | Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 17 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36 18 | Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 GTB6 (.NET CLR 3.5.30729) 19 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 20 | Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0 21 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1 22 | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36 23 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36 24 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36 25 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 26 | Mozilla/5.0 (Windows NT 5.1; rv:44.0) Gecko/20100101 Firefox/44.0 27 | Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36 28 | Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4 29 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1 30 | Mozilla/5.0 (Windows NT 6.3; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0 31 | Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0 32 | Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0 33 | Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0 34 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:28.0) Gecko/20100101 Firefox/28.0 35 | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) Qt/4.8.2 36 | Mozilla/5.0 (Windows NT 5.1; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 37 | Mozilla/3.0 (compatible; Indy Library) 38 | Mozilla/5.0 (Windows NT 5.1; rv:32.0) Gecko/20100101 Firefox/32.0 39 | Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 40 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 41 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1 42 | Mozilla/5.0 (Windows NT 6.3; WOW64; ReadSharp/6.3.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1736.2 Safari/537.36 OPR/20.0.1380.1 43 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36 44 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MRA 4.4 (build 01334); .NET CLR 1.1.4322) 45 | Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.9.0.1) Gecko/2008070208 46 | Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/29.0.1547.76 Safari/537.36 47 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.27+ (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27 48 | Mozilla/5.0 (Windows NT 5.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0/puffin 49 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E; MSN OptimizedIE8;NLNL) 50 | Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; WOW64; Trident/6.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729; InfoPath.3 GD 51 | ['Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; WOW64; Trident/6.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729; InfoPath.3 GD'] 52 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3 53 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 54 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; rv:11.0) like Gecko) 55 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en-US; .NET CLR 1.0.3328) 56 | Mozilla/5.0 (Windows NT x.y; rv:10.0) Gecko/20100101 Firefox/10.0 57 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0) 58 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 59 | Mozilla/37.0.2 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322) 60 | Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/) 61 | Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727) 62 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.60 Safari/537.1 63 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36 QQBrowser/3.8.3858.400 64 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0 65 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 66 | Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36 67 | Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html) 68 | Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.baidu.com) -------------------------------------------------------------------------------- /Yiyuyan/mimvp-打开网页.e: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/mimvp-打开网页.e -------------------------------------------------------------------------------- /Yiyuyan/mimvp-易语言基本语法.e: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/mimvp-易语言基本语法.e -------------------------------------------------------------------------------- /Yiyuyan/mimvp-易语言设置代理.e: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/mimvp-易语言设置代理.e -------------------------------------------------------------------------------- /Yiyuyan/mimvp-窗体中打开网页.e: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/mimvp-窗体中打开网页.e -------------------------------------------------------------------------------- /Yiyuyan/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/readme.txt -------------------------------------------------------------------------------- /Yiyuyan/精易模块.ec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/精易模块.ec -------------------------------------------------------------------------------- /Yiyuyan/精易模块5.36.ec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/精易模块5.36.ec -------------------------------------------------------------------------------- /Yiyuyan/超级模块7.3.reg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/超级模块7.3.reg -------------------------------------------------------------------------------- /Yiyuyan/超级模块8.0.ec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/Yiyuyan/超级模块8.0.ec -------------------------------------------------------------------------------- /cssjs/eyuyan-demo-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/cssjs/eyuyan-demo-1.png -------------------------------------------------------------------------------- /cssjs/eyuyan-demo-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/cssjs/eyuyan-demo-2.png -------------------------------------------------------------------------------- /cssjs/mimvp-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/cssjs/mimvp-logo.png -------------------------------------------------------------------------------- /cssjs/mimvp-proxy-demo-1-lang-proxy-protocol.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/cssjs/mimvp-proxy-demo-1-lang-proxy-protocol.png -------------------------------------------------------------------------------- /cssjs/mimvp-proxy-demo-2-lang-proxy-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mimvp/mimvp-proxy-demo/4960ed5691cbad57609e6ffd6b87c4acf523e267/cssjs/mimvp-proxy-demo-2-lang-proxy-demo.png -------------------------------------------------------------------------------- /cssjs/prism-default.css: -------------------------------------------------------------------------------- 1 | /* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+csharp+ruby+go+groovy+java+json+perl+php+python&plugins=line-numbers+toolbar+copy-to-clipboard */ 2 | /** 3 | * prism.js default theme for JavaScript, CSS and HTML 4 | * Based on dabblet (http://dabblet.com) 5 | * @author Lea Verou 6 | */ 7 | 8 | code[class*="language-"], 9 | pre[class*="language-"] { 10 | color: black; 11 | background: none; 12 | text-shadow: 0 1px white; 13 | font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; 14 | text-align: left; 15 | white-space: pre; 16 | word-spacing: normal; 17 | word-break: normal; 18 | word-wrap: normal; 19 | line-height: 1.5; 20 | 21 | -moz-tab-size: 4; 22 | -o-tab-size: 4; 23 | tab-size: 4; 24 | 25 | -webkit-hyphens: none; 26 | -moz-hyphens: none; 27 | -ms-hyphens: none; 28 | hyphens: none; 29 | } 30 | 31 | pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, 32 | code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { 33 | text-shadow: none; 34 | background: #b3d4fc; 35 | } 36 | 37 | pre[class*="language-"]::selection, pre[class*="language-"] ::selection, 38 | code[class*="language-"]::selection, code[class*="language-"] ::selection { 39 | text-shadow: none; 40 | background: #b3d4fc; 41 | } 42 | 43 | @media print { 44 | code[class*="language-"], 45 | pre[class*="language-"] { 46 | text-shadow: none; 47 | } 48 | } 49 | 50 | /* Code blocks */ 51 | pre[class*="language-"] { 52 | padding: 1em; 53 | margin: .5em 0; 54 | overflow: auto; 55 | } 56 | 57 | :not(pre) > code[class*="language-"], 58 | pre[class*="language-"] { 59 | background: #f5f2f0; 60 | } 61 | 62 | /* Inline code */ 63 | :not(pre) > code[class*="language-"] { 64 | padding: .1em; 65 | border-radius: .3em; 66 | white-space: normal; 67 | } 68 | 69 | .token.comment, 70 | .token.prolog, 71 | .token.doctype, 72 | .token.cdata { 73 | color: slategray; 74 | } 75 | 76 | .token.punctuation { 77 | color: #999; 78 | } 79 | 80 | .namespace { 81 | opacity: .7; 82 | } 83 | 84 | .token.property, 85 | .token.tag, 86 | .token.boolean, 87 | .token.number, 88 | .token.constant, 89 | .token.symbol, 90 | .token.deleted { 91 | color: #905; 92 | } 93 | 94 | .token.selector, 95 | .token.attr-name, 96 | .token.string, 97 | .token.char, 98 | .token.builtin, 99 | .token.inserted { 100 | color: #690; 101 | } 102 | 103 | .token.operator, 104 | .token.entity, 105 | .token.url, 106 | .language-css .token.string, 107 | .style .token.string { 108 | color: #a67f59; 109 | background: hsla(0, 0%, 100%, .5); 110 | } 111 | 112 | .token.atrule, 113 | .token.attr-value, 114 | .token.keyword { 115 | color: #07a; 116 | } 117 | 118 | .token.function { 119 | color: #DD4A68; 120 | } 121 | 122 | .token.regex, 123 | .token.important, 124 | .token.variable { 125 | color: #e90; 126 | } 127 | 128 | .token.important, 129 | .token.bold { 130 | font-weight: bold; 131 | } 132 | .token.italic { 133 | font-style: italic; 134 | } 135 | 136 | .token.entity { 137 | cursor: help; 138 | } 139 | 140 | pre.line-numbers { 141 | position: relative; 142 | padding-left: 3.8em; 143 | counter-reset: linenumber; 144 | } 145 | 146 | pre.line-numbers > code { 147 | position: relative; 148 | } 149 | 150 | .line-numbers .line-numbers-rows { 151 | position: absolute; 152 | pointer-events: none; 153 | top: 0; 154 | font-size: 100%; 155 | left: -3.8em; 156 | width: 3em; /* works for line-numbers below 1000 lines */ 157 | letter-spacing: -1px; 158 | border-right: 1px solid #999; 159 | 160 | -webkit-user-select: none; 161 | -moz-user-select: none; 162 | -ms-user-select: none; 163 | user-select: none; 164 | 165 | } 166 | 167 | .line-numbers-rows > span { 168 | pointer-events: none; 169 | display: block; 170 | counter-increment: linenumber; 171 | } 172 | 173 | .line-numbers-rows > span:before { 174 | content: counter(linenumber); 175 | color: #999; 176 | display: block; 177 | padding-right: 0.8em; 178 | text-align: right; 179 | } 180 | pre.code-toolbar { 181 | position: relative; 182 | } 183 | 184 | pre.code-toolbar > .toolbar { 185 | position: absolute; 186 | top: .3em; 187 | right: .2em; 188 | transition: opacity 0.3s ease-in-out; 189 | opacity: 0; 190 | } 191 | 192 | pre.code-toolbar:hover > .toolbar { 193 | opacity: 1; 194 | } 195 | 196 | pre.code-toolbar > .toolbar .toolbar-item { 197 | display: inline-block; 198 | } 199 | 200 | pre.code-toolbar > .toolbar a { 201 | cursor: pointer; 202 | } 203 | 204 | pre.code-toolbar > .toolbar button { 205 | background: none; 206 | border: 0; 207 | color: inherit; 208 | font: inherit; 209 | line-height: normal; 210 | overflow: visible; 211 | padding: 0; 212 | -webkit-user-select: none; /* for button */ 213 | -moz-user-select: none; 214 | -ms-user-select: none; 215 | } 216 | 217 | pre.code-toolbar > .toolbar a, 218 | pre.code-toolbar > .toolbar button, 219 | pre.code-toolbar > .toolbar span { 220 | color: #bbb; 221 | font-size: .8em; 222 | padding: 0 .5em; 223 | background: #f5f2f0; 224 | background: rgba(224, 224, 224, 0.2); 225 | box-shadow: 0 2px 0 0 rgba(0,0,0,0.2); 226 | border-radius: .5em; 227 | } 228 | 229 | pre.code-toolbar > .toolbar a:hover, 230 | pre.code-toolbar > .toolbar a:focus, 231 | pre.code-toolbar > .toolbar button:hover, 232 | pre.code-toolbar > .toolbar button:focus, 233 | pre.code-toolbar > .toolbar span:hover, 234 | pre.code-toolbar > .toolbar span:focus { 235 | color: inherit; 236 | text-decoration: none; 237 | } 238 | 239 | --------------------------------------------------------------------------------