├── local ├── ssl │ ├── serial │ ├── addroot.bat │ ├── certadm.dll │ ├── certutil.exe │ ├── ca.crt │ └── ca.key ├── py25.exe ├── taskbar.exe ├── proxy.ini ├── taskbar.py └── proxy.py ├── .gitignore ├── server ├── uploader.bat ├── app.yaml ├── fetch.py ├── fancy_urllib.py ├── appengine_rpc.py └── uploader.py ├── README.txt └── all.conf /local/ssl/serial: -------------------------------------------------------------------------------- 1 | 28 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo 3 | .appcfg_cookies -------------------------------------------------------------------------------- /local/ssl/addroot.bat: -------------------------------------------------------------------------------- 1 | %~dps0certutil.exe -f -addstore root "%~dp0ca.crt" -------------------------------------------------------------------------------- /local/py25.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qhgy/goagent/HEAD/local/py25.exe -------------------------------------------------------------------------------- /local/taskbar.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qhgy/goagent/HEAD/local/taskbar.exe -------------------------------------------------------------------------------- /local/ssl/certadm.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qhgy/goagent/HEAD/local/ssl/certadm.dll -------------------------------------------------------------------------------- /local/ssl/certutil.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qhgy/goagent/HEAD/local/ssl/certutil.exe -------------------------------------------------------------------------------- /server/uploader.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | set PYTHONOPTIMIZE=x 3 | "%~dp0..\local\py25.exe" uploader.py || pause 4 | @echo on -------------------------------------------------------------------------------- /server/app.yaml: -------------------------------------------------------------------------------- 1 | application: your_appid 2 | version: 1 3 | runtime: python 4 | api_version: 1 5 | 6 | handlers: 7 | - url: /fetch\.py 8 | script: fetch.py 9 | secure: optional 10 | -------------------------------------------------------------------------------- /local/ssl/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDUjCCAjoCAQAwDQYJKoZIhvcNAQEFBQAwbzEVMBMGA1UECxMMR29BZ2VudCBS 3 | b290MRAwDgYDVQQKEwdHb0FnZW50MRMwEQYDVQQDEwpHb0FnZW50IENBMREwDwYD 4 | VQQIEwhJbnRlcm5ldDELMAkGA1UEBhMCQ04xDzANBgNVBAcTBkNlcm5ldDAeFw0x 5 | MTA0MTUwMjI3MThaFw0zMTA0MTUwMjI3MThaMG8xFTATBgNVBAsTDEdvQWdlbnQg 6 | Um9vdDEQMA4GA1UEChMHR29BZ2VudDETMBEGA1UEAxMKR29BZ2VudCBDQTERMA8G 7 | A1UECBMISW50ZXJuZXQxCzAJBgNVBAYTAkNOMQ8wDQYDVQQHEwZDZXJuZXQwggEi 8 | MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDzIVKty2uPs/Q7By/5sCnaDhNo 9 | gF4+wivuTNj7k4XzOjyzaRxkIpd6RgVriwMyRQp0IcfjT3nOcTqP4DJUtZZHoiY+ 10 | XWxoKx5udjojr0CKLw5MfgD4lYsd6W+1eYmGQLnxhItil6KEduxbvs3Z3Fwub4lm 11 | 560nDE5Hyc04TjK4Rw/wIOrFpuSNX/gjhofdwoyeoVRQKNzhDAciQeoTPArRwFHd 12 | jmxwwtUDOz/LiRbB/HNxfritYtI6tz75augMDocbzlKRX5IRi3Tg2NmExmlFgw2S 13 | g0zk6mq0CkUkdDa7c6yX78jBOYru0w7+uA9z9YpDc/5AgdYVEZVt/3Ir9uRFAgMB 14 | AAEwDQYJKoZIhvcNAQEFBQADggEBAA8n+ghCdUitcQLa4nNBZYT9oYjFg+9aN9Fm 15 | hlqjJYn//ZpYiuoLchBJgAak7+EhlnskJG0P4qs0ejVkgE70ns2O2gGB0tKzZ71X 16 | MezWhT9L72UnVabCuWcHbVSA+RSD/DnUvOh29ZIfQUHJPl35v1mwtTe7wydEBguR 17 | ZxjE5wpD1rO8day3RdF3pIaFpwL1figLuKGd/1X25Eq/N6dzEDTtrSpuq+bn8TZw 18 | nfuifRJR/uCP9v8VUpOxlP2oIbk0+Xo+GD6qxoDIAst1Uoh2YkY39h8Hlj72yp2B 19 | dfMxTGPjCS+sCzkcEelmNFwVV0DBdyKtDQ7HxWkKu2Sa8ZpAITI= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /local/ssl/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEA8yFSrctrj7P0Owcv+bAp2g4TaIBePsIr7kzY+5OF8zo8s2kc 3 | ZCKXekYFa4sDMkUKdCHH4095znE6j+AyVLWWR6ImPl1saCsebnY6I69Aii8OTH4A 4 | +JWLHelvtXmJhkC58YSLYpeihHbsW77N2dxcLm+JZuetJwxOR8nNOE4yuEcP8CDq 5 | xabkjV/4I4aH3cKMnqFUUCjc4QwHIkHqEzwK0cBR3Y5scMLVAzs/y4kWwfxzcX64 6 | rWLSOrc++WroDA6HG85SkV+SEYt04NjZhMZpRYMNkoNM5OpqtApFJHQ2u3Osl+/I 7 | wTmK7tMO/rgPc/WKQ3P+QIHWFRGVbf9yK/bkRQIDAQABAoIBAEWQEDbXj+Piygsl 8 | iE15YNAZ3OW2bMCqD6Wz6RU55UZtMDbo6Q2hdBOw+xYFBRoZ9N67V5SrBZ/Sd734 9 | mI3yEphWRXjsg/rd82wJeaMCHoYq5n1uQ9rb9pzNUH/s0TNPS6RVlwfTeNVLrV6m 10 | ngEqcll64iZGPR2CANe7XnMwtIRSifHqTnHpGJ6O+3+hGr/6j2ZpjVrpvqNnxiZI 11 | bK5oFjsBgxM+eAu/IBQxUNDQFgYNBXMuHXqQhI8VxzBFm2xriWE0YteU08lu9xQg 12 | fHBLPHWgX4CvbhKs9kiEzdPgGYAHjIBTuK8sGS9aSb5CUNYU4QF/qmMjZfXH33om 13 | REvRiuECgYEA+xlFUEgq3PykhPJPdXuD+10yqFyh654YSJ8vjSh8B6/tmsVAI/1x 14 | 2ZLW379WK+pSEPh9KgMOqoZmjQSmfgrg9ocAcJP8Zxc65XbOFbGFw4TYRX87df3W 15 | HD7jFS9MZE1zq/2QJQWDPKmo1rOGhUO2aW5t5GuUn0Cbv1g8q9ToURkCgYEA9+A7 16 | 1fU83YigxaHJSTmjJ5W14wh/BDIV5Trh8Aoud330U6qWBgORQtL8XplFyZiy+p1T 17 | 9JYfd89Cx4bMXTK1zo3ROh4oe+SqQDbWSk5uRyDWnkitSBhPZUA8ALlYMCOwLpYx 18 | XulziiTztCgGCpJ3opvPbHy/U5DzUFgBvAf7tg0CgYEA0Oo6qEwTFaBCRbbRc57b 19 | tcTaBAhmVAJKlAmV5606XK78UtwwvID/O5YXnzuzt4AS3bnRcaXviuOd6VBoMdBd 20 | UeAK9p+5zhAe2ZIabyQvdfhOdKwiTc5vWTrddt/OgFmMlxm114eZpFxIdLITh0dK 21 | Org9SGJV2pZHv1Dr0c+npukCgYEAzCNig+mtD7FW3oxILkMGiDI2klxL5tOszpU4 22 | v6xS6lvT3Reu6BMGDaee6fWG0Okt9VGec98y2UPa9mGgaty5d/u5pQhzRN1kDPBc 23 | eOOw1GlJ9x9ZffdvY66L+/iolTS/Aw70Z/sRCWM3RVZ06z4GwudY4zq1gwfsKm3g 24 | N8/HT/0CgYEAjtX92NRDkjptqQvyoHx8Sna8f5UIFqzj+dmexajQad8lTerPO2GN 25 | d60pDS7sRLUaVqynJFVi2h+6UeRjzohlr4S/9+l/o1diCCRIPTucxR+gCUFmxfAQ 26 | 1OqDxY43Cup3cOdcu9+R3aTKa/yNuY7/5i9Sf8rUKiKEAAlQc3wKlOM= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /local/proxy.ini: -------------------------------------------------------------------------------- 1 | [listen] 2 | ip = 127.0.0.1 3 | port = 8087 4 | visible = 1 5 | 6 | [hosts] 7 | www.google.com:443 = 64.233.183.104|64.233.183.105|64.233.183.106|64.233.183.147|64.233.183.99|72.14.203.103|72.14.203.104|72.14.203.105|72.14.203.99|74.125.153.103|74.125.153.104|74.125.153.105|74.125.153.106|74.125.153.147|74.125.153.99|74.125.71.103|74.125.71.104|74.125.71.99 8 | encrypted.google.com:443 = 64.233.183.104|64.233.183.105|64.233.183.106|64.233.183.147|64.233.183.99|72.14.203.103|72.14.203.104|72.14.203.105|72.14.203.99|74.125.153.103|74.125.153.104|74.125.153.105|74.125.153.106|74.125.153.147|74.125.153.99|74.125.71.103|74.125.71.104|74.125.71.99 9 | mail.google.com:443 = 64.233.183.104|64.233.183.105|64.233.183.106|64.233.183.147|64.233.183.99|72.14.203.103|72.14.203.104|72.14.203.105|72.14.203.99|74.125.153.103|74.125.153.104|74.125.153.105|74.125.153.106|74.125.153.147|74.125.153.99|74.125.71.103|74.125.71.104|74.125.71.99 10 | appengine.google.com:443 = 64.233.183.104|64.233.183.105|64.233.183.106|64.233.183.147|64.233.183.99|72.14.203.103|72.14.203.104|72.14.203.105|72.14.203.99|74.125.153.103|74.125.153.104|74.125.153.105|74.125.153.106|74.125.153.147|74.125.153.99|74.125.71.103|74.125.71.104|74.125.71.99 11 | 12 | [gae] 13 | host = your_appid.appspot.com 14 | path = /fetch.py 15 | prefer = http 16 | verify = 1 17 | http = 203.208.37.99|203.208.39.99|203.208.37.104|203.208.39.104:80 18 | https = 209.85.143.83|209.85.146.19|209.85.148.17|209.85.153.83|209.85.157.83|209.85.175.17|209.85.195.83|209.85.225.17|209.85.227.17|209.85.229.18|66.102.11.83|66.102.13.17|66.102.13.19|74.125.159.17|74.125.224.55|74.125.225.22|74.125.226.119|74.125.226.183|74.125.227.22|74.125.227.56|74.125.229.117|74.125.230.119|74.125.230.120|74.125.230.150|74.125.230.85|74.125.232.119|74.125.232.120|74.125.233.23|74.125.235.21|74.125.235.53|74.125.235.54|74.125.235.87|74.125.237.55|74.125.39.19|74.125.71.17|74.125.71.83|74.125.77.18|74.125.79.18|74.125.79.19|74.125.79.83 19 | ipv6 = ipv6.google.com 20 | #proxy = http://www.google.cn:80 21 | 22 | 23 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | GoAgent FAQ https://github.com/phus/goagent 2 | 3 | Q: GoAgent是什么？ 4 | A: GoAgent是一个使用Python和Google Appengine SDK编写的代理软件。 5 | 6 | Q: 如何部署和使用GoAgent? 7 | A: 1.申请Google Appengine并创建appid 8 | 2.下载GoAgent https://github.com/phus/goagent/zipball/master 9 | 3.双击server\upload.bat,输入你的appid和你的用户名密码，上传服务端 10 | 4.把local\proxy.ini中的your_appid改成你申请到的appid 11 | 好了，现在你可以运行taskbar.exe启动代理了。 12 | 13 | Q: 如何最小化GoAgent那个黑乎乎的DOS窗口? 14 | A: 启动taskbar.exe之后托盘区会有GoAgent的图标，单击或者右击它就可以了。也可以编辑proxy.ini, 设置visible = 0 15 | 16 | Q: 我是Linux/Unix用户怎么办？ 17 | A: 上传完服务端并设置好proxy.ini之后，直接运行local/proxy.py即可。需要Python 2.5+和Python-OpenSSL这个包。 18 | 19 | Q: 既然已有WallProxy/GappProxy，为什么需要有GoAgent? 20 | A: WallProxy项目关闭了，GappProxy半年没更新。为了应对经常变化的网络状况，需要一个更新快的GoAgent。 21 | 22 | Q: 比WallProxy/GappProxy强在哪里？ 23 | A: 更新快，速度快，适应能力强。 24 | 25 | Q: 需要装Python或者Google Appenginge SDK后才能用GoAgent吗？ 26 | A: 完全不用，GoAgent是绿色软件哦。 27 | 28 | Q: GoAgent有哪些弱点？ 29 | A: 为了简单快速，GoAgent的数据没有强加密，使用的是head+hex/gzip格式来传输数据。 30 | 31 | Q: 为什么要叫GoAgent，而不叫GoProxy？ 32 | A: 一开始叫GoProxy的，后来Hewig说软件名字带有proxy字样不祥，于是就改成了GoAgent。 33 | 34 | Q: 为什么有时候GoAgent运行得好好的，突然出来一个502错误？ 35 | A: 有两种原因，1.配置错误，具体请看 http://65px.com/1993 ，2.网络出错，GoAgent此时会尝试重连,试试刷新一下浏览器就好了。 36 | 37 | Q: Firefox怎么不能登陆一些https网站? 38 | A: 打开FireFox->选项->高级->加密->查看证书->导入证书, 选择local\ssl\ca.crt, 勾选所有项，导入。现在的ca.crt来自于wallproxy 0.4.0，如果已经导入过了，尝试删除后或者新建一个profile再导入。 39 | 40 | Q: Chrome下如何使用GoAgent? 41 | A: Chrome可以安装proxy swithy插件，然后可以这样设置:图一: http://i.imgur.com/bJo1p.gif ,图二: http://i.imgur.com/aTH77.gif .注意，如果是用的ADSL或者VPN的话，需要在proxy swithy的Network中选中那个拨号连接。而且拨号连接必须是英文的(这个似乎是proxy swithy的limitation)。 42 | 43 | Q: 为什么一运行GoAgent后，py25.exe占用了40M内存？ 44 | A: GoAgent使用psyco1.6提速，所以内存占用有点多。如果你不希望使用这个机制的话，请下载这个py25.exe然后替换 https://github.com/phus/python-tools/blob/master/py25.exe?raw=true 45 | 46 | Q: 支持多个fetch server吗？ 47 | A: 目前GoAgent最新版是支持的，在proxy.ini中的[gae]项目下这样配置即可host=xxx.appspot.com|yyy.appspot.com|zzz.appspot.com 48 | 49 | Q: 如何得到GoAgent的源代码？ 50 | A: GoAgent的代码和程序是一起的，源代码就是运行程序。 51 | 52 | Q: 如何对GoAgent进行修改？ 53 | A: 客户端代码直接改local/proxy.py,改完重启taskbar.exe即可；服务端改server/fetch.py,改完用upload.bat上传即可。 54 | 55 | Q: 已做的工作和将要做的工作？ 56 | A: DONE: 57 | 1. 随机获取proxy.ini中配置的可用fetch ip,提高网络适应能力 58 | 2. 对于google的某些https域名，直接启用转发。 59 | 3. 移植了wallproxy的_RangeFetch，比较好的支持视频 60 | 4. 支持多个fetch server 61 | TODO: 62 | 2. 实现xmpp fetch 63 | 64 | Q: 有问题怎么办？ 65 | A: 请发信给我，我会把问题加到本页面的。 66 | -------------------------------------------------------------------------------- /all.conf: -------------------------------------------------------------------------------- 1 | ###################################################################### 2 | # By @逗bi极客 3 | # 2016.6.30 4 | # 夜市小灶 5 | # http://www.yeshigeek.com 6 | # Q群:817167 7 | # 🇯🇵 🇸🇬 🇰🇷 🇺🇸 🇨🇳 🇭🇰 🇼🇸 8 | # 更新日志、配置方法、高级运用，请进入博客查看! 9 | # 如果您喜欢，请在关注或转发时标注作者名称及地址，感谢您的支持！ 10 | # 4.11 更新路由表 11 | # 4.12 修复部分app不显示图片问题 12 | # 4.15 DOUYU更新,启动APP过滤测试,github,北京-移动-爱奇艺过滤增加 BY@笑熬浆糊-Mark 13 | # 河南-爱奇艺过滤增加BY@梁山伯伯阿 14 | # 4.18 更新风行规则感谢 By@Martking 15 | # 北京-联通-爱奇艺过滤增加 By@笑熬浆糊-Mark 16 | # 4.30 调整skip-proxy 17 | # 5.3 调整skip-proxy,删除P*规则,更新Appspot By@Chenxiaozhuang 18 | # 6.3 增加gedawang、shentu广告规则 By@夜懂白天 19 | # 湖北宜昌-电信 爱奇艺过滤增加，感谢 BY@呆萌二师弟 20 | # 6.4 修复优酷客户端的签到、网易客户端的签到问题 感谢 @LisonFan 21 | # 6.17 优化全能、精简版； 22 | # 重写APPLE规则； 23 | # LIVE提供 感谢@没有尾巴的猫@Forkazmodan； 24 | # Feng客户端广告过滤 感谢@撸大湿太@白天； 25 | # 北京-联通-爱奇艺过滤增加 感谢@xiaofengweilin 26 | # 成都-联通-爱奇艺过滤增加 感谢@风为裳夕相待 27 | # 6.19 修复视频APP开播下方面板问题； 28 | # 修复跳转问题； 29 | # 6.20 增加预留； 30 | # 6.22 美团增加; 31 | # 修复运营商劫持; 32 | # 重写删除; 33 | # 6.23 芒果修复闪退、假死问题； 34 | # 6.27 IPV6去除，修复Spotify 感谢@没有尾巴的猫 35 | # 6.30 移动4G环境页面过滤； 36 | # VSCO修复、DUOBAO修复 37 | ###################################################################### 38 | 39 | [General] 40 | loglevel = notify 41 | #ipv6 = true 42 | skip-proxy = 127.0.0.0/24, 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, 100.64.0.0/10, localhost, *.local, e.crashlytics.com 43 | #备份 44 | #skip-proxy = 127.0.0.0/24, 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, 100.64.0.0/10, localhost, *.local, e.crashlytics.com, ::ffff:0:0:0:0/1, ::ffff:128:0:0:0/1 45 | bypass-tun = 192.168.0.0/16, 10.0.0.0/8, 172.0.0.0/8, 127.0.0.0/24 46 | 47 | //国内常用DNS 48 | #dns-server = 223.6.6.6, 119.29.29.29, 223.5.5.5, 114.114.114.114 49 | 50 | # MAC版代理443配置 # 51 | #0.0.0.0为共享至网内其他机器使用 52 | interface = 0.0.0.0 53 | #port443指的是共享的http或https端口 54 | port = 8000 55 | #socks-port指的SS5的端口 56 | socks-port = 8001 57 | 58 | [Proxy] 59 | 🐈 DIRECT = direct 60 | 🇭🇰 HK-SERVER1 = custom,www.yeshigeek.com,443,rc4-md5,123456,https://down.qingjie.me:443/surge/ss.module,tcp-fast-open=true 61 | 🇭🇰 HK-SERVER2 = custom,www.yeshigeek.com,443,rc4-md5,123456,https://down.qingjie.me:443/surge/ss.module,tcp-fast-open=true 62 | 🇰🇷 KR-SERVER = custom,www.yeshigeek.com,443,rc4-md5,123456,https://down.qingjie.me:443/surge/ss.module,tcp-fast-open=true 63 | 🇯🇵 JP-SERVER = custom,www.yeshigeek.com,443,rc4-md5,123456,https://down.qingjie.me:443/surge/ss.module,tcp-fast-open=true 64 | 🇯🇵 SG-SERVER = custom,www.yeshigeek.com,443,rc4-md5,123456,https://down.qingjie.me:443/surge/ss.module,tcp-fast-open=true 65 | 🇺🇸 US-SERVER = custom,www.yeshigeek.com,443,rc4-md5,123456,https://down.qingjie.me:443/surge/ss.module,tcp-fast-open=true 66 | 67 | [Proxy Group] 68 | Proxy = select, 🐈 DIRECT, 🎲 CLUSTER, 🇭🇰 HK-SERVER1, 🇭🇰 HK-SERVER2, 🇰🇷 KR-SERVER, 🇯🇵 JP-SERVER, 🇺🇸 US-SERVER 69 | 🎲 CLUSTER = url-test, 🇭🇰 HK-SERVER1, 🇭🇰 HK-SERVER2, url = http://www.gstatic.com/generate_204, interval = 600, tolerance = 200, timeout = 5 70 | 71 | #[SSID Setting] 72 | #"WiFi SSID" suspend=false 73 | 74 | [Host] 75 | 76 | [Rule] 77 | 78 | FINAL,Proxy 79 | 80 | [URL Rewrite] 81 | -------------------------------------------------------------------------------- /local/taskbar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf-8 3 | 4 | # Creates a task-bar icon with balloon tip. Run from Python.exe to see the 5 | # messages printed. Right click for balloon tip. Double click to exit. 6 | # original version of this demo available at http://www.itamarst.org/software/ 7 | import pywintypes, win32api, win32con, win32gui, win32process 8 | import sys, os, ctypes 9 | 10 | WM_TASKBARNOTIFY = win32con.WM_USER+20 11 | WM_TASKBARNOTIFY_MENUITEM_SHOW = win32con.WM_USER + 21 12 | WM_TASKBARNOTIFY_MENUITEM_HIDE = win32con.WM_USER + 22 13 | WM_TASKBARNOTIFY_MENUITEM_EXIT = win32con.WM_USER + 23 14 | class Taskbar(object): 15 | def __init__(self, cmd, tooltip): 16 | self.cmd = cmd 17 | message_map = { 18 | win32con.WM_DESTROY: self.onDestroy, 19 | win32con.WM_COMMAND: self.onCommand, 20 | WM_TASKBARNOTIFY : self.onTaskbarNotify, 21 | } 22 | # Register the Window class. 23 | wc = win32gui.WNDCLASS() 24 | wc.hInstance = win32api.GetModuleHandle(None) 25 | wc.lpszClassName = "PythonTaskbarDemo" 26 | wc.style = win32con.CS_VREDRAW | win32con.CS_HREDRAW; 27 | wc.hCursor = win32gui.LoadCursor(0, win32con.IDC_ARROW) 28 | wc.hbrBackground = win32con.COLOR_WINDOW 29 | wc.lpfnWndProc = message_map # could also specify a wndproc. 30 | classAtom = win32gui.RegisterClass(wc) 31 | # Create the Window. 32 | style = win32con.WS_OVERLAPPED | win32con.WS_SYSMENU 33 | self.hwnd = win32gui.CreateWindow( classAtom, "Taskbar Demo", style, \ 34 | 0, 0, win32con.CW_USEDEFAULT, win32con.CW_USEDEFAULT, \ 35 | 0, 0, wc.hInstance, None) 36 | win32gui.UpdateWindow(self.hwnd) 37 | 38 | hProcess, hThread, dwProcessId, dwThreadId = win32process.CreateProcess(None, self.cmd, None, None, 0, 0, None, None, win32process.STARTUPINFO()) 39 | self.hProcess = hProcess 40 | try: 41 | hicon, small = win32gui.ExtractIconEx(win32api.GetModuleFileName(0), 0) 42 | win32gui.DestroyIcon(small[0]) 43 | #hicon = pywintypes.HANDLE(hicon[0]) 44 | hicon = hicon[0] 45 | except IndexError: 46 | hicon = win32gui.LoadIcon(0, win32con.IDI_APPLICATION) 47 | self.hicon = hicon 48 | self.tooltip = tooltip 49 | self.show() 50 | 51 | def show(self): 52 | """Display the taskbar icon""" 53 | flags = win32gui.NIF_ICON | win32gui.NIF_MESSAGE 54 | if self.tooltip is not None: 55 | flags |= win32gui.NIF_TIP 56 | nid = (self.hwnd, 0, flags, WM_TASKBARNOTIFY, self.hicon, self.tooltip) 57 | else: 58 | nid = (self.hwnd, 0, flags, WM_TASKBARNOTIFY, self.hicon) 59 | win32gui.Shell_NotifyIcon(win32gui.NIM_ADD, nid) 60 | self.visible = 1 61 | 62 | def hide(self): 63 | """Hide the taskbar icon""" 64 | if self.visible: 65 | nid = (self.hwnd, 0) 66 | win32gui.Shell_NotifyIcon(win32gui.NIM_DELETE, nid) 67 | self.visible = 0 68 | 69 | def onDestroy(self, hwnd, msg, wparam, lparam): 70 | self.hide() 71 | win32gui.PostQuitMessage(0) # Terminate the app. 72 | 73 | def onTaskbarNotify(self, hwnd, msg, wparam, lparam): 74 | if lparam == win32con.WM_LBUTTONUP: 75 | self.onClick() 76 | elif lparam == win32con.WM_LBUTTONDBLCLK: 77 | self.onDoubleClick() 78 | elif lparam == win32con.WM_RBUTTONUP: 79 | self.onRightClick() 80 | return 1 81 | 82 | def onCommand(self, hwnd, msg, wparam, lparam): 83 | nID = win32api.LOWORD(wparam) 84 | hwnd = ctypes.windll.kernel32.GetConsoleWindow() 85 | if nID == WM_TASKBARNOTIFY_MENUITEM_SHOW: 86 | win32gui.ShowWindow(hwnd, win32con.SW_SHOW|win32con.SW_MAXIMIZE) 87 | elif nID == WM_TASKBARNOTIFY_MENUITEM_HIDE: 88 | win32gui.ShowWindow(hwnd, win32con.SW_HIDE) 89 | elif nID == WM_TASKBARNOTIFY_MENUITEM_EXIT: 90 | win32process.TerminateProcess(self.hProcess, 0) 91 | self.hide() 92 | sys.exit(0) 93 | return 1 94 | 95 | def onClick(self): 96 | hwnd = ctypes.windll.kernel32.GetConsoleWindow() 97 | v = ctypes.windll.user32.IsWindowVisible(hwnd) 98 | win32gui.ShowWindow(hwnd, {1:0,0:1}[v]) 99 | win32gui.SetForegroundWindow(hwnd) 100 | 101 | def onDoubleClick(self): 102 | pass 103 | 104 | def onRightClick(self): 105 | menu = win32gui.CreatePopupMenu() 106 | win32gui.AppendMenu(menu, win32con.MF_STRING, WM_TASKBARNOTIFY_MENUITEM_SHOW, u'显示') 107 | win32gui.AppendMenu(menu, win32con.MF_STRING, WM_TASKBARNOTIFY_MENUITEM_HIDE, u'隐藏') 108 | win32gui.AppendMenu(menu, win32con.MF_STRING, WM_TASKBARNOTIFY_MENUITEM_EXIT, u'退出') 109 | pos = win32gui.GetCursorPos() 110 | win32gui.SetForegroundWindow(self.hwnd) 111 | win32gui.TrackPopupMenu(menu, win32con.TPM_LEFTALIGN, pos[0], pos[1], 0, self.hwnd, None) 112 | win32gui.PostMessage(self.hwnd, win32con.WM_NULL, 0, 0) 113 | 114 | if __name__=='__main__': 115 | os.chdir(os.path.dirname(__file__)) 116 | os.environ['PYTHONOPTIMIZE'] = 'x' 117 | t = Taskbar('py25.exe proxy.py', 'GoAgent beta') 118 | win32gui.PumpMessages() -------------------------------------------------------------------------------- /server/fetch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Based on GAppProxy by Du XiaoGang 4 | # Based on WallProxy 0.4.0 by hexieshe 5 | 6 | __version__ = 'beta' 7 | __author__ = 'phus.lu@gmail.com' 8 | 9 | import zlib, logging, time, re, struct 10 | from google.appengine.ext import webapp 11 | from google.appengine.ext import db 12 | from google.appengine.ext.webapp.util import run_wsgi_app 13 | from google.appengine.api import urlfetch 14 | from google.appengine.runtime import apiproxy_errors 15 | 16 | def encode_data(dic): 17 | return '&'.join('%s=%s' % (k, str(v).encode('hex')) for k, v in dic.iteritems()) 18 | 19 | def decode_data(qs): 20 | return dict((k, v.decode('hex')) for k, v in (x.split('=') for x in qs.split('&'))) 21 | 22 | class MainHandler(webapp.RequestHandler): 23 | #FRS_Headers = ('', 'content-length', 'keep-alive', 'host', 'vary', 'via', 'x-forwarded-for', 24 | # 'proxy-authorization', 'proxy-connection', 'upgrade') 25 | FRP_Headers = ('', 'x-google-cache-control', 'via') 26 | Fetch_Max = 3 27 | Fetch_MaxSize = 512*1000 28 | Deadline = (15, 30) 29 | 30 | def sendResponse(self, status_code, headers, content='', method='', url=''): 31 | self.response.headers['Content-Type'] = 'application/octet-stream' 32 | contentType = headers.get('content-type', '').lower() 33 | 34 | headers = encode_data(headers) 35 | # Build send-data 36 | rdata = '%s%s%s' % (struct.pack('>3I', status_code, len(headers), len(content)), headers, content) 37 | if contentType.startswith(('text', 'application')): 38 | data = zlib.compress(rdata) 39 | data = '1'+data if len(rdata)>len(data) else '0'+rdata 40 | else: 41 | data = '0' + rdata 42 | if status_code == 555: 43 | logging.warning('Response: "%s %s" %s' % (method, url, content)) 44 | else: 45 | logging.debug('Response: "%s %s" %d %d/%d/%d' % (method, url, status_code, len(content), len(rdata), len(data))) 46 | return self.response.out.write(data) 47 | 48 | def sendNotify(self, status_code, content, method='', url='', fullContent=False): 49 | if not fullContent and status_code!=555: 50 | content = '

Fetch Server Info

Code: %d

' \ 51 | '

Message: %s

' % (status_code, content) 52 | headers = {'server':'GoAgent GAE/%s' % __version__, 'content-type':'text/html', 'content-length':len(content)} 53 | self.sendResponse(status_code, headers, content, method, url) 54 | 55 | def post(self): 56 | request = decode_data(zlib.decompress(self.request.body)) 57 | 58 | method = request.get('method', 'GET') 59 | fetch_method = getattr(urlfetch, method, '') 60 | if not fetch_method: 61 | return self.sendNotify(555, 'Invalid Method', method) 62 | 63 | url = request.get('url', '') 64 | if not url.startswith('http'): 65 | return self.sendNotify(555, 'Unsupported Scheme', method, url) 66 | 67 | payload = request.get('payload', '') 68 | deadline = MainHandler.Deadline[1 if payload else 0] 69 | 70 | fetch_range = 'bytes=0-%d' % (MainHandler.Fetch_MaxSize - 1) 71 | rangeFetch = False 72 | headers = {} 73 | for line in request.get('headers', '').splitlines(): 74 | kv = line.split(':', 1) 75 | if len(kv) != 2: 76 | continue 77 | key = kv[0].strip().lower() 78 | value = kv[1].strip() 79 | #if key in MainHandler.FRS_Headers: 80 | # continue 81 | if key == 'rangefetch': 82 | rangeFetch = True 83 | continue 84 | if key =='range' and not rangeFetch: 85 | m = re.search(r'(\d+)?-(\d+)?', value) 86 | if not m: 87 | continue 88 | m = [u and int(u) for u in m.groups()] 89 | if m[0] is None and m[1] is None: 90 | continue 91 | if m[0] is None and m[1] > MainHandler.Fetch_MaxSize: 92 | m[1] = 1023 93 | elif m[1] is None or m[1]-m[0]+1 > MainHandler.Fetch_MaxSize: 94 | m[1] = MainHandler.Fetch_MaxSize - 1 + m[0] 95 | fetch_range = ('bytes=%s-%s' % (m[0] if m[0] is not None else '', m[0] if m[1] is not None else '')) 96 | headers[key] = value 97 | headers['Connection'] = 'close' 98 | 99 | for i in range(MainHandler.Fetch_Max): 100 | try: 101 | response = urlfetch.fetch(url, payload, fetch_method, headers, False, False, deadline) 102 | #if method=='GET' and len(response.content)>0x1000000: 103 | # raise urlfetch.ResponseTooLargeError(None) 104 | break 105 | except apiproxy_errors.OverQuotaError, e: 106 | time.sleep(2) 107 | except urlfetch.InvalidURLError, e: 108 | return self.sendNotify(555, 'Invalid URL: %s' % e, method, url) 109 | except urlfetch.ResponseTooLargeError, e: 110 | if method == 'GET': 111 | deadline = MainHandler.Deadline[1] 112 | if not rangeFetch: 113 | headers['Range'] = fetch_range 114 | else: 115 | return self.sendNotify(555, 'Response Too Large: %s' % e, method, url) 116 | except Exception, e: 117 | if i==0 and method=='GET': 118 | deadline = MainHandler.Deadline[1] 119 | if not rangeFetch: 120 | headers['Range'] = fetch_range 121 | else: 122 | return self.sendNotify(555, 'Urlfetch error: %s' % e, method, url) 123 | 124 | for k in MainHandler.FRP_Headers: 125 | if k in response.headers: 126 | del response.headers[k] 127 | if 'set-cookie' in response.headers: 128 | scs = response.headers['set-cookie'].split(', ') 129 | cookies = [] 130 | i = -1 131 | for sc in scs: 132 | if re.match(r'[^ =]+ ', sc): 133 | try: 134 | cookies[i] = '%s, %s' % (cookies[i], sc) 135 | except IndexError: 136 | pass 137 | else: 138 | cookies.append(sc) 139 | i += 1 140 | response.headers['set-cookie'] = '\r\nSet-Cookie: '.join(cookies) 141 | response.headers['connection'] = 'close' 142 | return self.sendResponse(response.status_code, response.headers, response.content, method, url) 143 | 144 | def get(self): 145 | self.response.headers['Content-Type'] = 'text/html; charset=utf-8' 146 | self.response.out.write( \ 147 | ''' 148 | 149 | 150 | 151 | GoAgent %(version)s on GAE/已经在工作了 152 | 153 | 154 | 155 | 156 | 159 | 160 | 161 | 164 | 165 | 166 | 169 | 170 | 171 | 174 | 175 |
157 |

GoAgent %(version)s on GAE/已经在工作了

 158 |
162 | GoAgent是一个开源的HTTP Proxy软件,使用Python编写,运行于Google App Engine平台上. 163 |
167 | 更多相关介绍,请参考GoAgent项目主页. 168 |
172 | Powered by Google App Engine 173 |
176 | 177 | 178 | ''' % dict(version=__version__)) 179 | 180 | def main(): 181 | application = webapp.WSGIApplication([(r'/fetch.py', MainHandler)], debug=True) 182 | run_wsgi_app(application) 183 | 184 | if __name__ == '__main__': 185 | main() -------------------------------------------------------------------------------- /server/fancy_urllib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.4 2 | # 3 | # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software 4 | # Foundation; All Rights Reserved 5 | 6 | """A HTTPSConnection/Handler with additional proxy and cert validation features. 7 | 8 | In particular, monkey patches in Python r74203 to provide support for CONNECT 9 | proxies and adds SSL cert validation if the ssl module is present. 10 | """ 11 | 12 | __author__ = "{frew,nick.johnson}@google.com (Fred Wulff and Nick Johnson)" 13 | 14 | import base64 15 | import httplib 16 | import logging 17 | import re 18 | import socket 19 | import urllib2 20 | 21 | from urllib import splittype 22 | from urllib import splituser 23 | from urllib import splitpasswd 24 | 25 | class InvalidCertificateException(httplib.HTTPException): 26 | """Raised when a certificate is provided with an invalid hostname.""" 27 | 28 | def __init__(self, host, cert, reason): 29 | """Constructor. 30 | 31 | Args: 32 | host: The hostname the connection was made to. 33 | cert: The SSL certificate (as a dictionary) the host returned. 34 | """ 35 | httplib.HTTPException.__init__(self) 36 | self.host = host 37 | self.cert = cert 38 | self.reason = reason 39 | 40 | def __str__(self): 41 | return ('Host %s returned an invalid certificate (%s): %s\n' 42 | 'To learn more, see ' 43 | 'http://code.google.com/appengine/kb/general.html#rpcssl' % 44 | (self.host, self.reason, self.cert)) 45 | 46 | def can_validate_certs(): 47 | """Return True if we have the SSL package and can validate certificates.""" 48 | try: 49 | import ssl 50 | return True 51 | except ImportError: 52 | return False 53 | 54 | def _create_fancy_connection(tunnel_host=None, key_file=None, 55 | cert_file=None, ca_certs=None): 56 | # This abomination brought to you by the fact that 57 | # the HTTPHandler creates the connection instance in the middle 58 | # of do_open so we need to add the tunnel host to the class. 59 | 60 | class PresetProxyHTTPSConnection(httplib.HTTPSConnection): 61 | """An HTTPS connection that uses a proxy defined by the enclosing scope.""" 62 | 63 | def __init__(self, *args, **kwargs): 64 | httplib.HTTPSConnection.__init__(self, *args, **kwargs) 65 | 66 | self._tunnel_host = tunnel_host 67 | if tunnel_host: 68 | logging.debug("Creating preset proxy https conn: %s", tunnel_host) 69 | 70 | self.key_file = key_file 71 | self.cert_file = cert_file 72 | self.ca_certs = ca_certs 73 | try: 74 | import ssl 75 | if self.ca_certs: 76 | self.cert_reqs = ssl.CERT_REQUIRED 77 | else: 78 | self.cert_reqs = ssl.CERT_NONE 79 | except ImportError: 80 | pass 81 | 82 | def _tunnel(self): 83 | self._set_hostport(self._tunnel_host, None) 84 | logging.info("Connecting through tunnel to: %s:%d", 85 | self.host, self.port) 86 | self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port)) 87 | response = self.response_class(self.sock, strict=self.strict, 88 | method=self._method) 89 | (_, code, message) = response._read_status() 90 | 91 | if code != 200: 92 | self.close() 93 | raise socket.error, "Tunnel connection failed: %d %s" % ( 94 | code, message.strip()) 95 | 96 | while True: 97 | line = response.fp.readline() 98 | if line == "\r\n": 99 | break 100 | 101 | def _get_valid_hosts_for_cert(self, cert): 102 | """Returns a list of valid host globs for an SSL certificate. 103 | 104 | Args: 105 | cert: A dictionary representing an SSL certificate. 106 | Returns: 107 | list: A list of valid host globs. 108 | """ 109 | if 'subjectAltName' in cert: 110 | return [x[1] for x in cert['subjectAltName'] if x[0].lower() == 'dns'] 111 | else: 112 | # Return a list of commonName fields 113 | return [x[0][1] for x in cert['subject'] 114 | if x[0][0].lower() == 'commonname'] 115 | 116 | def _validate_certificate_hostname(self, cert, hostname): 117 | """Validates that a given hostname is valid for an SSL certificate. 118 | 119 | Args: 120 | cert: A dictionary representing an SSL certificate. 121 | hostname: The hostname to test. 122 | Returns: 123 | bool: Whether or not the hostname is valid for this certificate. 124 | """ 125 | hosts = self._get_valid_hosts_for_cert(cert) 126 | for host in hosts: 127 | # Convert the glob-style hostname expression (eg, '*.google.com') into a 128 | # valid regular expression. 129 | host_re = host.replace('.', '\.').replace('*', '[^.]*') 130 | if re.search('^%s$' % (host_re,), hostname, re.I): 131 | return True 132 | return False 133 | 134 | 135 | def connect(self): 136 | # TODO(frew): When we drop support for <2.6 (in the far distant future), 137 | # change this to socket.create_connection. 138 | self.sock = _create_connection((self.host, self.port)) 139 | 140 | if self._tunnel_host: 141 | self._tunnel() 142 | 143 | # ssl and FakeSocket got deprecated. Try for the new hotness of wrap_ssl, 144 | # with fallback. 145 | try: 146 | import ssl 147 | self.sock = ssl.wrap_socket(self.sock, 148 | keyfile=self.key_file, 149 | certfile=self.cert_file, 150 | ca_certs=self.ca_certs, 151 | cert_reqs=self.cert_reqs) 152 | 153 | if self.cert_reqs & ssl.CERT_REQUIRED: 154 | cert = self.sock.getpeercert() 155 | hostname = self.host.split(':', 0)[0] 156 | if not self._validate_certificate_hostname(cert, hostname): 157 | raise InvalidCertificateException(hostname, cert, 158 | 'hostname mismatch') 159 | except ImportError: 160 | ssl = socket.ssl(self.sock, 161 | keyfile=self.key_file, 162 | certfile=self.cert_file) 163 | self.sock = httplib.FakeSocket(self.sock, ssl) 164 | 165 | return PresetProxyHTTPSConnection 166 | 167 | 168 | # Here to end of _create_connection copied wholesale from Python 2.6"s socket.py 169 | _GLOBAL_DEFAULT_TIMEOUT = object() 170 | 171 | 172 | def _create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT): 173 | """Connect to *address* and return the socket object. 174 | 175 | Convenience function. Connect to *address* (a 2-tuple ``(host, 176 | port)``) and return the socket object. Passing the optional 177 | *timeout* parameter will set the timeout on the socket instance 178 | before attempting to connect. If no *timeout* is supplied, the 179 | global default timeout setting returned by :func:`getdefaulttimeout` 180 | is used. 181 | """ 182 | 183 | msg = "getaddrinfo returns an empty list" 184 | host, port = address 185 | for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): 186 | af, socktype, proto, canonname, sa = res 187 | sock = None 188 | try: 189 | sock = socket.socket(af, socktype, proto) 190 | if timeout is not _GLOBAL_DEFAULT_TIMEOUT: 191 | sock.settimeout(timeout) 192 | sock.connect(sa) 193 | return sock 194 | 195 | except socket.error, msg: 196 | if sock is not None: 197 | sock.close() 198 | 199 | raise socket.error, msg 200 | 201 | 202 | class FancyRequest(urllib2.Request): 203 | """A request that allows the use of a CONNECT proxy.""" 204 | 205 | def __init__(self, *args, **kwargs): 206 | urllib2.Request.__init__(self, *args, **kwargs) 207 | self._tunnel_host = None 208 | self._key_file = None 209 | self._cert_file = None 210 | self._ca_certs = None 211 | 212 | def set_proxy(self, host, type): 213 | saved_type = None 214 | if self.get_type() == "https" and not self._tunnel_host: 215 | self._tunnel_host = self.get_host() 216 | saved_type = self.get_type() 217 | urllib2.Request.set_proxy(self, host, type) 218 | 219 | if saved_type: 220 | # Don't set self.type, we want to preserve the 221 | # type for tunneling. 222 | self.type = saved_type 223 | 224 | def set_ssl_info(self, key_file=None, cert_file=None, ca_certs=None): 225 | self._key_file = key_file 226 | self._cert_file = cert_file 227 | self._ca_certs = ca_certs 228 | 229 | 230 | class FancyProxyHandler(urllib2.ProxyHandler): 231 | """A ProxyHandler that works with CONNECT-enabled proxies.""" 232 | 233 | # Taken verbatim from /usr/lib/python2.5/urllib2.py 234 | def _parse_proxy(self, proxy): 235 | """Return (scheme, user, password, host/port) given a URL or an authority. 236 | 237 | If a URL is supplied, it must have an authority (host:port) component. 238 | According to RFC 3986, having an authority component means the URL must 239 | have two slashes after the scheme: 240 | 241 | >>> _parse_proxy('file:/ftp.example.com/') 242 | Traceback (most recent call last): 243 | ValueError: proxy URL with no authority: 'file:/ftp.example.com/' 244 | 245 | The first three items of the returned tuple may be None. 246 | 247 | Examples of authority parsing: 248 | 249 | >>> _parse_proxy('proxy.example.com') 250 | (None, None, None, 'proxy.example.com') 251 | >>> _parse_proxy('proxy.example.com:3128') 252 | (None, None, None, 'proxy.example.com:3128') 253 | 254 | The authority component may optionally include userinfo (assumed to be 255 | username:password): 256 | 257 | >>> _parse_proxy('joe:password@proxy.example.com') 258 | (None, 'joe', 'password', 'proxy.example.com') 259 | >>> _parse_proxy('joe:password@proxy.example.com:3128') 260 | (None, 'joe', 'password', 'proxy.example.com:3128') 261 | 262 | Same examples, but with URLs instead: 263 | 264 | >>> _parse_proxy('http://proxy.example.com/') 265 | ('http', None, None, 'proxy.example.com') 266 | >>> _parse_proxy('http://proxy.example.com:3128/') 267 | ('http', None, None, 'proxy.example.com:3128') 268 | >>> _parse_proxy('http://joe:password@proxy.example.com/') 269 | ('http', 'joe', 'password', 'proxy.example.com') 270 | >>> _parse_proxy('http://joe:password@proxy.example.com:3128') 271 | ('http', 'joe', 'password', 'proxy.example.com:3128') 272 | 273 | Everything after the authority is ignored: 274 | 275 | >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128') 276 | ('ftp', 'joe', 'password', 'proxy.example.com') 277 | 278 | Test for no trailing '/' case: 279 | 280 | >>> _parse_proxy('http://joe:password@proxy.example.com') 281 | ('http', 'joe', 'password', 'proxy.example.com') 282 | 283 | """ 284 | scheme, r_scheme = splittype(proxy) 285 | if not r_scheme.startswith("/"): 286 | # authority 287 | scheme = None 288 | authority = proxy 289 | else: 290 | # URL 291 | if not r_scheme.startswith("//"): 292 | raise ValueError("proxy URL with no authority: %r" % proxy) 293 | # We have an authority, so for RFC 3986-compliant URLs (by ss 3. 294 | # and 3.3.), path is empty or starts with '/' 295 | end = r_scheme.find("/", 2) 296 | if end == -1: 297 | end = None 298 | authority = r_scheme[2:end] 299 | userinfo, hostport = splituser(authority) 300 | if userinfo is not None: 301 | user, password = splitpasswd(userinfo) 302 | else: 303 | user = password = None 304 | return scheme, user, password, hostport 305 | 306 | def proxy_open(self, req, proxy, type): 307 | # This block is copied wholesale from Python2.6 urllib2. 308 | # It is idempotent, so the superclass method call executes as normal 309 | # if invoked. 310 | orig_type = req.get_type() 311 | proxy_type, user, password, hostport = self._parse_proxy(proxy) 312 | if proxy_type is None: 313 | proxy_type = orig_type 314 | if user and password: 315 | user_pass = "%s:%s" % (urllib2.unquote(user), urllib2.unquote(password)) 316 | creds = base64.b64encode(user_pass).strip() 317 | # Later calls overwrite earlier calls for the same header 318 | req.add_header("Proxy-authorization", "Basic " + creds) 319 | hostport = urllib2.unquote(hostport) 320 | req.set_proxy(hostport, proxy_type) 321 | # This condition is the change 322 | if orig_type == "https": 323 | return None 324 | 325 | return urllib2.ProxyHandler.proxy_open(self, req, proxy, type) 326 | 327 | 328 | class FancyHTTPSHandler(urllib2.HTTPSHandler): 329 | """An HTTPSHandler that works with CONNECT-enabled proxies.""" 330 | 331 | def do_open(self, http_class, req): 332 | # Intentionally very specific so as to opt for false negatives 333 | # rather than false positives. 334 | try: 335 | return urllib2.HTTPSHandler.do_open( 336 | self, 337 | _create_fancy_connection(req._tunnel_host, 338 | req._key_file, 339 | req._cert_file, 340 | req._ca_certs), 341 | req) 342 | except urllib2.URLError, url_error: 343 | try: 344 | import ssl 345 | if (type(url_error.reason) == ssl.SSLError and 346 | url_error.reason.args[0] == 1): 347 | # Display the reason to the user. Need to use args for python2.5 348 | # compat. 349 | raise InvalidCertificateException(req.host, '', 350 | url_error.reason.args[1]) 351 | except ImportError: 352 | pass 353 | 354 | raise url_error 355 | 356 | 357 | # We have to implement this so that we persist the tunneling behavior 358 | # through redirects. 359 | class FancyRedirectHandler(urllib2.HTTPRedirectHandler): 360 | """A redirect handler that persists CONNECT-enabled proxy information.""" 361 | 362 | def redirect_request(self, req, *args, **kwargs): 363 | new_req = urllib2.HTTPRedirectHandler.redirect_request( 364 | self, req, *args, **kwargs) 365 | # Same thing as in our set_proxy implementation, but in this case 366 | # we"ve only got a Request to work with, so it was this or copy 367 | # everything over piecemeal. 368 | if hasattr(req, "_tunnel_host") and isinstance(new_req, urllib2.Request): 369 | if new_req.get_type() == "https": 370 | new_req._tunnel_host = new_req.get_host() 371 | new_req.set_proxy(req.host, "https") 372 | new_req.type = "https" 373 | new_req._key_file = req._key_file 374 | new_req._cert_file = req._cert_file 375 | new_req._ca_certs = req._ca_certs 376 | 377 | return new_req 378 | -------------------------------------------------------------------------------- /server/appengine_rpc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2007 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | """Tool for performing authenticated RPCs against App Engine.""" 19 | 20 | 21 | import cookielib 22 | import fancy_urllib 23 | import logging 24 | import os 25 | import re 26 | import socket 27 | import sys 28 | import urllib 29 | import urllib2 30 | 31 | logger = logging.getLogger('google.appengine.tools.appengine_rpc') 32 | 33 | def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform): 34 | """Returns a 'User-agent' token for the host system platform. 35 | 36 | Args: 37 | os_module, sys_module, platform: Used for testing. 38 | 39 | Returns: 40 | String containing the platform token for the host system. 41 | """ 42 | if hasattr(sys_module, "getwindowsversion"): 43 | windows_version = sys_module.getwindowsversion() 44 | version_info = ".".join(str(i) for i in windows_version[:4]) 45 | return platform + "/" + version_info 46 | elif hasattr(os_module, "uname"): 47 | uname = os_module.uname() 48 | return "%s/%s" % (uname[0], uname[2]) 49 | else: 50 | return "unknown" 51 | 52 | def HttpRequestToString(req, include_data=True): 53 | """Converts a urllib2.Request to a string. 54 | 55 | Args: 56 | req: urllib2.Request 57 | Returns: 58 | Multi-line string representing the request. 59 | """ 60 | 61 | headers = "" 62 | for header in req.header_items(): 63 | headers += "%s: %s\n" % (header[0], header[1]) 64 | 65 | template = ("%(method)s %(selector)s %(type)s/1.1\n" 66 | "Host: %(host)s\n" 67 | "%(headers)s") 68 | if include_data: 69 | template = template + "\n%(data)s" 70 | 71 | return template % { 72 | 'method' : req.get_method(), 73 | 'selector' : req.get_selector(), 74 | 'type' : req.get_type().upper(), 75 | 'host' : req.get_host(), 76 | 'headers': headers, 77 | 'data': req.get_data(), 78 | } 79 | 80 | class ClientLoginError(urllib2.HTTPError): 81 | """Raised to indicate there was an error authenticating with ClientLogin.""" 82 | 83 | def __init__(self, url, code, msg, headers, args): 84 | urllib2.HTTPError.__init__(self, url, code, msg, headers, None) 85 | self.args = args 86 | self.reason = args["Error"] 87 | 88 | def read(self): 89 | return '%d %s: %s' % (self.code, self.msg, self.reason) 90 | 91 | 92 | class AbstractRpcServer(object): 93 | """Provides a common interface for a simple RPC server.""" 94 | 95 | def __init__(self, host, auth_function, user_agent, source, 96 | host_override=None, extra_headers=None, save_cookies=False, 97 | auth_tries=3, account_type=None, debug_data=True, secure=True): 98 | """Creates a new HttpRpcServer. 99 | 100 | Args: 101 | host: The host to send requests to. 102 | auth_function: A function that takes no arguments and returns an 103 | (email, password) tuple when called. Will be called if authentication 104 | is required. 105 | user_agent: The user-agent string to send to the server. Specify None to 106 | omit the user-agent header. 107 | source: The source to specify in authentication requests. 108 | host_override: The host header to send to the server (defaults to host). 109 | extra_headers: A dict of extra headers to append to every request. Values 110 | supplied here will override other default headers that are supplied. 111 | save_cookies: If True, save the authentication cookies to local disk. 112 | If False, use an in-memory cookiejar instead. Subclasses must 113 | implement this functionality. Defaults to False. 114 | auth_tries: The number of times to attempt auth_function before failing. 115 | account_type: One of GOOGLE, HOSTED_OR_GOOGLE, or None for automatic. 116 | debug_data: Whether debugging output should include data contents. 117 | """ 118 | if secure: 119 | self.scheme = "https" 120 | else: 121 | self.scheme = "http" 122 | self.host = host 123 | self.host_override = host_override 124 | self.auth_function = auth_function 125 | self.source = source 126 | self.authenticated = False 127 | self.auth_tries = auth_tries 128 | self.debug_data = debug_data 129 | 130 | self.account_type = account_type 131 | 132 | self.extra_headers = {} 133 | if user_agent: 134 | self.extra_headers["User-Agent"] = user_agent 135 | if extra_headers: 136 | self.extra_headers.update(extra_headers) 137 | 138 | self.save_cookies = save_cookies 139 | self.cookie_jar = cookielib.MozillaCookieJar() 140 | self.opener = self._GetOpener() 141 | if self.host_override: 142 | logger.info("Server: %s; Host: %s", self.host, self.host_override) 143 | else: 144 | logger.info("Server: %s", self.host) 145 | 146 | if ((self.host_override and self.host_override == "localhost") or 147 | self.host == "localhost" or self.host.startswith("localhost:")): 148 | self._DevAppServerAuthenticate() 149 | 150 | def _GetOpener(self): 151 | """Returns an OpenerDirector for making HTTP requests. 152 | 153 | Returns: 154 | A urllib2.OpenerDirector object. 155 | """ 156 | raise NotImplemented() 157 | 158 | def _CreateRequest(self, url, data=None): 159 | """Creates a new urllib request.""" 160 | req = fancy_urllib.FancyRequest(url, data=data) 161 | if self.host_override: 162 | req.add_header("Host", self.host_override) 163 | for key, value in self.extra_headers.iteritems(): 164 | req.add_header(key, value) 165 | return req 166 | 167 | def _GetAuthToken(self, email, password): 168 | """Uses ClientLogin to authenticate the user, returning an auth token. 169 | 170 | Args: 171 | email: The user's email address 172 | password: The user's password 173 | 174 | Raises: 175 | ClientLoginError: If there was an error authenticating with ClientLogin. 176 | HTTPError: If there was some other form of HTTP error. 177 | 178 | Returns: 179 | The authentication token returned by ClientLogin. 180 | """ 181 | account_type = self.account_type 182 | if not account_type: 183 | if (self.host.split(':')[0].endswith(".google.com") 184 | or (self.host_override 185 | and self.host_override.split(':')[0].endswith(".google.com"))): 186 | account_type = "HOSTED_OR_GOOGLE" 187 | else: 188 | account_type = "GOOGLE" 189 | data = { 190 | "Email": email, 191 | "Passwd": password, 192 | "service": "ah", 193 | "source": self.source, 194 | "accountType": account_type 195 | } 196 | 197 | req = self._CreateRequest( 198 | url="https://www.google.com/accounts/ClientLogin", 199 | data=urllib.urlencode(data)) 200 | try: 201 | response = self.opener.open(req) 202 | response_body = response.read() 203 | response_dict = dict(x.split("=") 204 | for x in response_body.split("\n") if x) 205 | return response_dict["Auth"] 206 | except urllib2.HTTPError, e: 207 | if e.code == 403: 208 | body = e.read() 209 | response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) 210 | raise ClientLoginError(req.get_full_url(), e.code, e.msg, 211 | e.headers, response_dict) 212 | else: 213 | raise 214 | 215 | def _GetAuthCookie(self, auth_token): 216 | """Fetches authentication cookies for an authentication token. 217 | 218 | Args: 219 | auth_token: The authentication token returned by ClientLogin. 220 | 221 | Raises: 222 | HTTPError: If there was an error fetching the authentication cookies. 223 | """ 224 | continue_location = "http://localhost/" 225 | args = {"continue": continue_location, "auth": auth_token} 226 | login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah") 227 | req = self._CreateRequest("%s://%s%s/login?%s" % 228 | (self.scheme, self.host, login_path, 229 | urllib.urlencode(args))) 230 | try: 231 | response = self.opener.open(req) 232 | except urllib2.HTTPError, e: 233 | response = e 234 | if (response.code != 302 or 235 | response.info()["location"] != continue_location): 236 | raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, 237 | response.headers, response.fp) 238 | self.authenticated = True 239 | 240 | def _Authenticate(self): 241 | """Authenticates the user. 242 | 243 | The authentication process works as follows: 244 | 1) We get a username and password from the user 245 | 2) We use ClientLogin to obtain an AUTH token for the user 246 | (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). 247 | 3) We pass the auth token to /_ah/login on the server to obtain an 248 | authentication cookie. If login was successful, it tries to redirect 249 | us to the URL we provided. 250 | 251 | If we attempt to access the upload API without first obtaining an 252 | authentication cookie, it returns a 401 response and directs us to 253 | authenticate ourselves with ClientLogin. 254 | """ 255 | for unused_i in range(self.auth_tries): 256 | credentials = self.auth_function() 257 | try: 258 | auth_token = self._GetAuthToken(credentials[0], credentials[1]) 259 | except ClientLoginError, e: 260 | if e.reason == "BadAuthentication": 261 | print >>sys.stderr, "Invalid username or password." 262 | continue 263 | if e.reason == "CaptchaRequired": 264 | print >>sys.stderr, ( 265 | "Please go to\n" 266 | "https://www.google.com/accounts/DisplayUnlockCaptcha\n" 267 | "and verify you are a human. Then try again.") 268 | break 269 | if e.reason == "NotVerified": 270 | print >>sys.stderr, "Account not verified." 271 | break 272 | if e.reason == "TermsNotAgreed": 273 | print >>sys.stderr, "User has not agreed to TOS." 274 | break 275 | if e.reason == "AccountDeleted": 276 | print >>sys.stderr, "The user account has been deleted." 277 | break 278 | if e.reason == "AccountDisabled": 279 | print >>sys.stderr, "The user account has been disabled." 280 | break 281 | if e.reason == "ServiceDisabled": 282 | print >>sys.stderr, ("The user's access to the service has been " 283 | "disabled.") 284 | break 285 | if e.reason == "ServiceUnavailable": 286 | print >>sys.stderr, "The service is not available; try again later." 287 | break 288 | raise 289 | self._GetAuthCookie(auth_token) 290 | return 291 | 292 | def _DevAppServerAuthenticate(self): 293 | """Authenticates the user on the dev_appserver.""" 294 | pass 295 | 296 | def Send(self, request_path, payload="", 297 | content_type="application/octet-stream", 298 | timeout=None, 299 | **kwargs): 300 | """Sends an RPC and returns the response. 301 | 302 | Args: 303 | request_path: The path to send the request to, eg /api/appversion/create. 304 | payload: The body of the request, or None to send an empty request. 305 | content_type: The Content-Type header to use. 306 | timeout: timeout in seconds; default None i.e. no timeout. 307 | (Note: for large requests on OS X, the timeout doesn't work right.) 308 | kwargs: Any keyword arguments are converted into query string parameters. 309 | 310 | Returns: 311 | The response body, as a string. 312 | """ 313 | old_timeout = socket.getdefaulttimeout() 314 | socket.setdefaulttimeout(timeout) 315 | try: 316 | tries = 0 317 | auth_tried = False 318 | while True: 319 | tries += 1 320 | args = dict(kwargs) 321 | url = "%s://%s%s?%s" % (self.scheme, self.host, request_path, 322 | urllib.urlencode(args)) 323 | req = self._CreateRequest(url=url, data=payload) 324 | req.add_header("Content-Type", content_type) 325 | req.add_header("X-appcfg-api-version", "1") 326 | try: 327 | logger.debug('Sending HTTP request:\n%s', 328 | HttpRequestToString(req, include_data=self.debug_data)) 329 | f = self.opener.open(req) 330 | response = f.read() 331 | f.close() 332 | return response 333 | except urllib2.HTTPError, e: 334 | logger.debug("Got http error, this is try #%s", tries) 335 | if tries > self.auth_tries: 336 | raise 337 | elif e.code == 401: 338 | if auth_tried: 339 | raise 340 | auth_tried = True 341 | self._Authenticate() 342 | elif e.code >= 500 and e.code < 600: 343 | continue 344 | elif e.code == 302: 345 | if auth_tried: 346 | raise 347 | auth_tried = True 348 | loc = e.info()["location"] 349 | logger.debug("Got 302 redirect. Location: %s", loc) 350 | if loc.startswith("https://www.google.com/accounts/ServiceLogin"): 351 | self._Authenticate() 352 | elif re.match(r"https://www.google.com/a/[a-z0-9.-]+/ServiceLogin", 353 | loc): 354 | self.account_type = os.getenv("APPENGINE_RPC_HOSTED_LOGIN_TYPE", 355 | "HOSTED") 356 | self._Authenticate() 357 | elif loc.startswith("http://%s/_ah/login" % (self.host,)): 358 | self._DevAppServerAuthenticate() 359 | else: 360 | raise 361 | else: 362 | raise 363 | finally: 364 | socket.setdefaulttimeout(old_timeout) 365 | 366 | 367 | class HttpRpcServer(AbstractRpcServer): 368 | """Provides a simplified RPC-style interface for HTTP requests.""" 369 | 370 | DEFAULT_COOKIE_FILE_PATH = os.path.join(os.path.dirname(__file__), '.appcfg_cookies') 371 | 372 | def __init__(self, *args, **kwargs): 373 | self.certpath = os.path.normpath(os.path.join( 374 | os.path.dirname(__file__), '..', '..', '..', 'lib', 'cacerts', 375 | 'cacerts.txt')) 376 | self.cert_file_available = os.path.exists(self.certpath) 377 | super(HttpRpcServer, self).__init__(*args, **kwargs) 378 | 379 | def _CreateRequest(self, url, data=None): 380 | """Creates a new urllib request.""" 381 | req = super(HttpRpcServer, self)._CreateRequest(url, data) 382 | if self.cert_file_available and fancy_urllib.can_validate_certs(): 383 | req.set_ssl_info(ca_certs=self.certpath) 384 | return req 385 | 386 | def _Authenticate(self): 387 | """Save the cookie jar after authentication.""" 388 | if self.cert_file_available and not fancy_urllib.can_validate_certs(): 389 | logger.warn("""ssl module not found. 390 | Without the ssl module, the identity of the remote host cannot be verified, and 391 | connections may NOT be secure. To fix this, please install the ssl module from 392 | http://pypi.python.org/pypi/ssl . 393 | To learn more, see http://code.google.com/appengine/kb/general.html#rpcssl .""") 394 | super(HttpRpcServer, self)._Authenticate() 395 | if self.cookie_jar.filename is not None and self.save_cookies: 396 | logger.info("Saving authentication cookies to %s", 397 | self.cookie_jar.filename) 398 | self.cookie_jar.save() 399 | 400 | def _GetOpener(self): 401 | """Returns an OpenerDirector that supports cookies and ignores redirects. 402 | 403 | Returns: 404 | A urllib2.OpenerDirector object. 405 | """ 406 | opener = urllib2.OpenerDirector() 407 | opener.add_handler(fancy_urllib.FancyProxyHandler()) 408 | opener.add_handler(urllib2.UnknownHandler()) 409 | opener.add_handler(urllib2.HTTPHandler()) 410 | opener.add_handler(urllib2.HTTPDefaultErrorHandler()) 411 | opener.add_handler(fancy_urllib.FancyHTTPSHandler()) 412 | opener.add_handler(urllib2.HTTPErrorProcessor()) 413 | 414 | if self.save_cookies: 415 | self.cookie_jar.filename = os.path.expanduser( 416 | HttpRpcServer.DEFAULT_COOKIE_FILE_PATH) 417 | 418 | if os.path.exists(self.cookie_jar.filename): 419 | try: 420 | self.cookie_jar.load() 421 | self.authenticated = True 422 | logger.info("Loaded authentication cookies from %s", 423 | self.cookie_jar.filename) 424 | except (OSError, IOError, cookielib.LoadError), e: 425 | logger.debug("Could not load authentication cookies; %s: %s", 426 | e.__class__.__name__, e) 427 | self.cookie_jar.filename = None 428 | else: 429 | try: 430 | fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600) 431 | os.close(fd) 432 | except (OSError, IOError), e: 433 | logger.debug("Could not create authentication cookies file; %s: %s", 434 | e.__class__.__name__, e) 435 | self.cookie_jar.filename = None 436 | 437 | opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) 438 | return opener -------------------------------------------------------------------------------- /server/uploader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2007 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | import hashlib 19 | import urllib 20 | import urllib2 21 | import sys 22 | import os 23 | import re 24 | import time 25 | import getpass 26 | import appengine_rpc 27 | 28 | LIST_DELIMITER = '\n' 29 | TUPLE_DELIMITER = '|' 30 | MAX_BATCH_SIZE = 1000000 31 | MAX_BATCH_COUNT = 100 32 | MAX_BATCH_FILE_SIZE = 200000 33 | BATCH_OVERHEAD = 500 34 | BASE_DIR = "." 35 | 36 | verbosity = 1 37 | 38 | def GetUserCredentials(): 39 | """Prompts the user for a username and password.""" 40 | email = None 41 | if email is None: 42 | email = raw_input('Email: ') 43 | password_prompt = 'Password for %s: ' % email 44 | password = getpass.getpass(password_prompt) 45 | return (email, password) 46 | 47 | class UploadBatcher(object): 48 | """Helper to batch file uploads.""" 49 | 50 | def __init__(self, what, app_id, version, server): 51 | """Constructor. 52 | 53 | Args: 54 | what: Either 'file' or 'blob' or 'errorblob' indicating what kind of 55 | objects this batcher uploads. Used in messages and URLs. 56 | app_id: The application ID. 57 | version: The application version string. 58 | server: The RPC server. 59 | """ 60 | assert what in ('file', 'blob', 'errorblob'), repr(what) 61 | self.what = what 62 | self.app_id = app_id 63 | self.version = version 64 | self.server = server 65 | self.single_url = '/api/appversion/add' + what 66 | self.batch_url = self.single_url + 's' 67 | self.batching = True 68 | self.batch = [] 69 | self.batch_size = 0 70 | 71 | def SendBatch(self): 72 | """Send the current batch on its way. 73 | 74 | If successful, resets self.batch and self.batch_size. 75 | 76 | Raises: 77 | HTTPError with code=404 if the server doesn't support batching. 78 | """ 79 | boundary = 'boundary' 80 | parts = [] 81 | for path, payload, mime_type in self.batch: 82 | while boundary in payload: 83 | boundary += '%04x' % random.randint(0, 0xffff) 84 | assert len(boundary) < 80, 'Unexpected error, please try again.' 85 | part = '\n'.join(['', 'X-Appcfg-File: %s' % urllib.quote(path), 'X-Appcfg-Hash: %s' % _Hash(payload), 'Content-Type: %s' % mime_type, 'Content-Length: %d' % len(payload), 'Content-Transfer-Encoding: 8bit', '', payload, ]) 86 | parts.append(part) 87 | parts.insert(0, 'MIME-Version: 1.0\n' 'Content-Type: multipart/mixed; boundary="%s"\n' '\n' 'This is a message with multiple parts in MIME format.' % boundary) 88 | parts.append('--\n') 89 | delimiter = '\n--%s' % boundary 90 | payload = delimiter.join(parts) 91 | self.server.Send(self.batch_url, payload=payload, content_type='message/rfc822', app_id=self.app_id, version=self.version) 92 | self.batch = [] 93 | self.batch_size = 0 94 | 95 | def SendSingleFile(self, path, payload, mime_type): 96 | """Send a single file on its way.""" 97 | self.server.Send(self.single_url, payload=payload, content_type=mime_type, path=path, app_id=self.app_id, version=self.version) 98 | 99 | def Flush(self): 100 | """Flush the current batch. 101 | 102 | This first attempts to send the batch as a single request; if that 103 | fails because the server doesn't support batching, the files are 104 | sent one by one, and self.batching is reset to False. 105 | 106 | At the end, self.batch and self.batch_size are reset. 107 | """ 108 | if not self.batch: 109 | return 110 | try: 111 | self.SendBatch() 112 | except urllib2.HTTPError, err: 113 | if err.code != 404: 114 | raise 115 | self.batching = False 116 | for path, payload, mime_type in self.batch: 117 | self.SendSingleFile(path, payload, mime_type) 118 | self.batch = [] 119 | self.batch_size = 0 120 | 121 | def AddToBatch(self, path, payload, mime_type): 122 | """Batch a file, possibly flushing first, or perhaps upload it directly. 123 | 124 | Args: 125 | path: The name of the file. 126 | payload: The contents of the file. 127 | mime_type: The MIME Content-type of the file, or None. 128 | 129 | If mime_type is None, application/octet-stream is substituted. 130 | """ 131 | if not mime_type: 132 | mime_type = 'application/octet-stream' 133 | size = len(payload) 134 | if size <= MAX_BATCH_FILE_SIZE: 135 | if (len(self.batch) >= MAX_BATCH_COUNT or self.batch_size + size > MAX_BATCH_SIZE): 136 | self.Flush() 137 | if self.batching: 138 | self.batch.append((path, payload, mime_type)) 139 | self.batch_size += size + BATCH_OVERHEAD 140 | return 141 | self.SendSingleFile(path, payload, mime_type) 142 | 143 | def StatusUpdate(msg): 144 | """Print a status message to stderr. 145 | 146 | If 'verbosity' is greater than 0, print the message. 147 | 148 | Args: 149 | msg: The string to print. 150 | """ 151 | if verbosity > 0: 152 | print >>sys.stderr, msg 153 | 154 | def _Hash(content): 155 | """Compute the hash of the content. 156 | 157 | Args: 158 | content: The data to hash as a string. 159 | 160 | Returns: 161 | The string representation of the hash. 162 | """ 163 | m = hashlib.sha1() 164 | m.update(content) 165 | h = m.hexdigest() 166 | return '%s_%s_%s_%s_%s' % (h[0:8], h[8:16], h[16:24], h[24:32], h[32:40]) 167 | 168 | def BuildClonePostBody(file_tuples): 169 | """Build the post body for the /api/clone{files,blobs,errorblobs} urls. 170 | 171 | Args: 172 | file_tuples: A list of tuples. Each tuple should contain the entries 173 | appropriate for the endpoint in question. 174 | 175 | Returns: 176 | A string containing the properly delimited tuples. 177 | """ 178 | file_list = [] 179 | for tup in file_tuples: 180 | path = tup[0] 181 | tup = tup[1:] 182 | file_list.append(TUPLE_DELIMITER.join([path] + list(tup))) 183 | return LIST_DELIMITER.join(file_list) 184 | 185 | def RetryWithBackoff(initial_delay, backoff_factor, max_delay, max_tries, callable_func): 186 | """Calls a function multiple times, backing off more and more each time. 187 | 188 | Args: 189 | initial_delay: Initial delay after first try, in seconds. 190 | backoff_factor: Delay will be multiplied by this factor after each try. 191 | max_delay: Max delay factor. 192 | max_tries: Maximum number of tries. 193 | callable_func: The method to call, will pass no arguments. 194 | 195 | Returns: 196 | True if the function succeded in one of its tries. 197 | 198 | Raises: 199 | Whatever the function raises--an exception will immediately stop retries. 200 | """ 201 | delay = initial_delay 202 | if callable_func(): 203 | return True 204 | while max_tries > 1: 205 | StatusUpdate('Will check again in %s seconds.' % delay) 206 | time.sleep(delay) 207 | delay *= backoff_factor 208 | if max_delay and delay > max_delay: 209 | delay = max_delay 210 | max_tries -= 1 211 | if callable_func(): 212 | return True 213 | return False 214 | 215 | class AppVersionUpload(object): 216 | """Provides facilities to upload a new appversion to the hosting service. 217 | 218 | Attributes: 219 | server: The AbstractRpcServer to use for the upload. 220 | config: The AppInfoExternal object derived from the app.yaml file. 221 | app_id: The application string from 'config'. 222 | version: The version string from 'config'. 223 | files: A dictionary of files to upload to the server, mapping path to 224 | hash of the file contents. 225 | in_transaction: True iff a transaction with the server has started. 226 | An AppVersionUpload can do only one transaction at a time. 227 | deployed: True iff the Deploy method has been called. 228 | """ 229 | 230 | def __init__(self, server, app_id): 231 | """Creates a new AppVersionUpload. 232 | 233 | Args: 234 | server: The RPC server to use. Should be an instance of HttpRpcServer or 235 | TestRpcServer. 236 | """ 237 | self.server = server 238 | self.app_id = app_id 239 | self.version = 1 240 | self.yaml = re.sub(r'(?m)application: \w+', 'application: '+app_id, open('app.yaml', 'rb').read()) 241 | print self.yaml 242 | self.files = {} 243 | self.in_transaction = False 244 | self.deployed = False 245 | self.batching = True 246 | self.file_batcher = UploadBatcher('file', self.app_id, self.version, self.server) 247 | 248 | def AddFile(self, path, file_handle): 249 | """Adds the provided file to the list to be pushed to the server. 250 | 251 | Args: 252 | path: The path the file should be uploaded as. 253 | file_handle: A stream containing data to upload. 254 | """ 255 | assert not self.in_transaction, 'Already in a transaction.' 256 | assert file_handle is not None 257 | pos = file_handle.tell() 258 | content_hash = _Hash(file_handle.read()) 259 | file_handle.seek(pos, 0) 260 | self.files[path] = content_hash 261 | 262 | def Begin(self): 263 | """Begins the transaction, returning a list of files that need uploading. 264 | 265 | All calls to AddFile must be made before calling Begin(). 266 | 267 | Returns: 268 | A list of pathnames for files that should be uploaded using UploadFile() 269 | before Commit() can be called. 270 | """ 271 | assert not self.in_transaction, 'Already in a transaction.' 272 | StatusUpdate('Initiating update.') 273 | self.server.Send('/api/appversion/create', app_id=self.app_id, version=self.version, payload=self.yaml) 274 | self.in_transaction = True 275 | files_to_clone = [] 276 | for path, content_hash in self.files.iteritems(): 277 | files_to_clone.append((path, content_hash)) 278 | files_to_upload = {} 279 | 280 | def CloneFiles(url, files, file_type): 281 | """Sends files to the given url. 282 | 283 | Args: 284 | url: the server URL to use. 285 | files: a list of files 286 | file_type: the type of the files 287 | """ 288 | if not files: 289 | return 290 | result = self.server.Send(url, app_id=self.app_id, version=self.version, payload=BuildClonePostBody(files)) 291 | if result: 292 | files_to_upload.update(dict((f, self.files[f]) for f in result.split(LIST_DELIMITER))) 293 | 294 | CloneFiles('/api/appversion/clonefiles', files_to_clone, 'application') 295 | self.files = files_to_upload 296 | return sorted(files_to_upload.iterkeys()) 297 | 298 | def UploadFile(self, path, file_handle): 299 | """Uploads a file to the hosting service. 300 | 301 | Must only be called after Begin(). 302 | The path provided must be one of those that were returned by Begin(). 303 | 304 | Args: 305 | path: The path the file is being uploaded as. 306 | file_handle: A file-like object containing the data to upload. 307 | 308 | Raises: 309 | KeyError: The provided file is not amongst those to be uploaded. 310 | """ 311 | assert self.in_transaction, 'Begin() must be called before UploadFile().' 312 | if path not in self.files: 313 | raise KeyError('File \'%s\' is not in the list of files to be uploaded.' % path) 314 | del self.files[path] 315 | self.file_batcher.AddToBatch(path, file_handle.read(), None) 316 | 317 | def Commit(self): 318 | """Commits the transaction, making the new app version available. 319 | 320 | All the files returned by Begin() must have been uploaded with UploadFile() 321 | before Commit() can be called. 322 | 323 | This tries the new 'deploy' method; if that fails it uses the old 'commit'. 324 | 325 | Raises: 326 | Exception: Some required files were not uploaded. 327 | """ 328 | assert self.in_transaction, 'Begin() must be called before Commit().' 329 | if self.files: 330 | raise Exception('Not all required files have been uploaded.') 331 | try: 332 | self.Deploy() 333 | if not RetryWithBackoff(1, 2, 60, 20, self.IsReady): 334 | raise Exception('Version not ready.') 335 | self.StartServing() 336 | except urllib2.HTTPError, e: 337 | if e.code != 404: 338 | raise 339 | StatusUpdate('Closing update.') 340 | self.server.Send('/api/appversion/commit', app_id=self.app_id, version=self.version) 341 | self.in_transaction = False 342 | 343 | def Deploy(self): 344 | """Deploys the new app version but does not make it default. 345 | 346 | All the files returned by Begin() must have been uploaded with UploadFile() 347 | before Deploy() can be called. 348 | 349 | Raises: 350 | Exception: Some required files were not uploaded. 351 | """ 352 | assert self.in_transaction, 'Begin() must be called before Deploy().' 353 | if self.files: 354 | raise Exception('Not all required files have been uploaded.') 355 | StatusUpdate('Deploying new version.') 356 | self.server.Send('/api/appversion/deploy', app_id=self.app_id, version=self.version) 357 | self.deployed = True 358 | 359 | def IsReady(self): 360 | """Check if the new app version is ready to serve traffic. 361 | 362 | Raises: 363 | Exception: Deploy has not yet been called. 364 | 365 | Returns: 366 | True if the server returned the app is ready to serve. 367 | """ 368 | assert self.deployed, 'Deploy() must be called before IsReady().' 369 | StatusUpdate('Checking if new version is ready to serve.') 370 | result = self.server.Send('/api/appversion/isready', app_id=self.app_id, version=self.version) 371 | return result == '1' 372 | 373 | def StartServing(self): 374 | """Start serving with the newly created version. 375 | 376 | Raises: 377 | Exception: Deploy has not yet been called. 378 | """ 379 | assert self.deployed, 'Deploy() must be called before IsReady().' 380 | StatusUpdate('Closing update: new version is ready to start serving.') 381 | self.server.Send('/api/appversion/startserving', app_id=self.app_id, version=self.version) 382 | self.in_transaction = False 383 | 384 | def Rollback(self): 385 | """Rolls back the transaction if one is in progress.""" 386 | if not self.in_transaction: 387 | return 388 | StatusUpdate('Rolling back the update.') 389 | self.server.Send('/api/appversion/rollback', app_id=self.app_id, version=self.version) 390 | self.in_transaction = False 391 | self.files = {} 392 | 393 | def DoUpload(self): 394 | """Uploads a new appversion with the given config and files to the server.""" 395 | self.AddFile("fetch.py", open("%s/fetch.py" % BASE_DIR, "r")) 396 | try: 397 | missing_files = self.Begin() 398 | if missing_files: 399 | StatusUpdate('Uploading %d files and blobs.' % len(missing_files)) 400 | num_files = 0 401 | for missing_file in missing_files: 402 | file_handle = open("%s/%s" % (BASE_DIR, missing_file), "r") 403 | try: 404 | self.UploadFile(missing_file, file_handle) 405 | finally: 406 | file_handle.close() 407 | num_files += 1 408 | self.file_batcher.Flush() 409 | StatusUpdate('Uploaded %d files and blobs' % num_files) 410 | self.Commit() 411 | except: 412 | self.Rollback() 413 | raise 414 | 415 | def main(): 416 | if len(sys.argv) == 2 and sys.argv[1] != "update" and sys.argv[1] != "rollback": 417 | print "Usage: %s [update|rollback]" % sys.argv[0] 418 | return 419 | rpc_server = appengine_rpc.HttpRpcServer("appengine.google.com", GetUserCredentials, "GAppProxy Uploader", "0.0.1", host_override=None, save_cookies=True, auth_tries=3, account_type='HOSTED_OR_GOOGLE', secure=True) 420 | if 'AppID' in os.environ: 421 | AppID = os.environ['AppID'] 422 | else: 423 | AppID = raw_input("AppID: ") 424 | appversion = AppVersionUpload(rpc_server, AppID) 425 | if len(sys.argv) == 2 and sys.argv[1] == "rollback": 426 | appversion.in_transaction = True 427 | appversion.Rollback() 428 | else: # update 429 | appversion.DoUpload() 430 | time.sleep(30) 431 | 432 | if __name__ == "__main__": 433 | main() 434 | -------------------------------------------------------------------------------- /local/proxy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Based on GAppProxy 2.0.0 by Du XiaoGang 4 | # Based on WallProxy 0.4.0 by hexieshe 5 | 6 | import sys, os, re, time 7 | import errno, zlib, random, struct, traceback 8 | import httplib, urllib2, urlparse, socket, select 9 | import thread, BaseHTTPServer, SocketServer 10 | import ConfigParser 11 | import ssl, OpenSSL 12 | 13 | __version__ = 'beta' 14 | __author__ = 'phus.lu@gmail.com' 15 | 16 | class RandomTCPConnection(object): 17 | '''random tcp connection class''' 18 | CONNECT_COUNT = 5 19 | CONNECT_TIMEOUT = 3 20 | def __init__(self, hosts, port): 21 | self.socket = None 22 | self.__socs = [] 23 | self.connect(hosts, port) 24 | def connect(self, hosts, port): 25 | if len(hosts) > RandomTCPConnection.CONNECT_COUNT: 26 | hosts = random.Random().sample(hosts, RandomTCPConnection.CONNECT_COUNT) 27 | for host in hosts: 28 | soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 29 | soc.setblocking(0) 30 | err = soc.connect_ex((host, port)) 31 | self.__socs.append(soc) 32 | (_, outs, _) = select.select([], self.__socs, [], RandomTCPConnection.CONNECT_TIMEOUT) 33 | if outs: 34 | self.socket = outs[0] 35 | self.socket.setblocking(1) 36 | def close(self): 37 | for soc in self.__socs: 38 | try: 39 | soc.close() 40 | except: 41 | pass 42 | 43 | class Common(object): 44 | '''global config module, based on GappProxy 2.0.0''' 45 | FILENAME = sys.argv[1] if len(sys.argv) == 2 and os.path.isfile(os.sys.argv[1]) else os.path.splitext(__file__)[0] + '.ini' 46 | ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P