├── README ├── MANIFEST.in ├── dev_requirements.txt ├── img_test.png ├── MANIFEST ├── tox.ini ├── .gitignore ├── setup.py ├── README.md ├── tests.py └── tornadohttpclient.py /README: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include img_test.png 2 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | tornado 2 | pycurl 3 | -------------------------------------------------------------------------------- /img_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coldnight/tornadohttpclient/HEAD/img_test.png -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | README 3 | img_test.png 4 | setup.py 5 | tornadohttpclient.py 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py27,py3,py35 3 | 4 | [testenv] 5 | deps = -rdev_requirements.txt 6 | nose 7 | commands=nosetests 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.o 3 | *.sw[po] 4 | *~ 5 | .DS_Store 6 | *.log 7 | *.tmp 8 | .idea 9 | tags 10 | .#* 11 | *.db 12 | logs/ 13 | conf.py 14 | build/ 15 | dist/ 16 | .tox/ 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | from setuptools import setup 4 | 5 | 6 | requires = ['tornado', 'pycurl'] 7 | 8 | setup( 9 | name='tornadohttpclient', 10 | version='1.1.3', 11 | description='Asynchronous http client.', 12 | long_description=("TornadoHTTPClient 对" 13 | "tornado.curl_httpclient.CurlAsyncHTTPClient的封装, " 14 | "支持cookie"), 15 | author='cold', 16 | author_email='wh_linux@126.com', 17 | url='http://www.linuxzen.com', 18 | py_modules=['tornadohttpclient'], 19 | scripts=[], 20 | install_requires=requires, 21 | license='Apache 2.0', 22 | platforms='any', 23 | classifiers=[ 24 | 'Development Status :: 3 - Alpha', 25 | "Intended Audience :: Developers", 26 | 'License :: OSI Approved :: Apache Software License', 27 | 'Topic :: Internet :: WWW/HTTP', 28 | 'Programming Language :: Python :: 2.7', 29 | 'Programming Language :: Python :: 3', 30 | 'Programming Language :: Python :: 3.0', 31 | 'Programming Language :: Python :: 3.3', 32 | 'Programming Language :: Python :: 3.4', 33 | 'Programming Language :: Python :: 3.5', 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | TornadoHTTPClient 对tornado.curl_httpclient.CurlAsyncHTTPClient的封装, 支持cookie 2 | 3 | ## 安装 4 | ```bash 5 | python setup.py install 6 | ``` 7 | 8 | ## 教程 9 | ### GET 10 | TornadoHTTPClient的get方法可以发起一个get请求 11 | ```python 12 | from tornado import gen 13 | from tornado.ioloop import IOLoop 14 | 15 | from tornadohttpclient import TornadoHTTPClient 16 | 17 | # 实例化 18 | http = TornadoHTTPClient() 19 | 20 | @gen.coroutine 21 | def get(): 22 | # 发出get请求 23 | response = yield http.get("http://www.linuxzen.com") 24 | print(response.body) 25 | 26 | IOLoop.instance().run_sync(get) 27 | ``` 28 | 29 | ### 上传文件 30 | 31 | ```python 32 | from tornado import gen 33 | from tornado.ioloop import IOLoop 34 | 35 | from tornadohttpclient import TornadoHTTPClient 36 | 37 | # 实例化 38 | http = TornadoHTTPClient() 39 | 40 | @gen.coroutine 41 | def run(): 42 | response = yield http.upload("http://paste.linuxzen.com", "img", 43 | "img_test.png", callback=callback) 44 | print("打开图片链接", end=" ") 45 | print(response.effective_url) 46 | 47 | IOLoop.instance().run_sync(run) 48 | ``` 49 | 50 | ### 给请求传递参数 51 | TornadoHTTPClient 的 `get`/`post`方法的第二个参数`data`可以定义请求时传递的参数`data`的类型为字典或者`((key, value), )`类型的元组或列表,例如使用百度搜索`TornadoHTTPClient` 52 | ```python 53 | from tornado import gen 54 | from tornado.ioloop import IOLoop 55 | 56 | from tornadohttpclient import TornadoHTTPClient 57 | 58 | # 实例化 59 | http = TornadoHTTPClient() 60 | 61 | @gen.coroutine 62 | def run(): 63 | response = yield http.get("http://www.baidu.com/s", (("wd", "tornado"),)) 64 | print(response.effective_url) 65 | 66 | IOLoop.instance().run_sync(run) 67 | ``` 68 | 69 | 以上也使用与POST方法, 比如登录网站 70 | ```python 71 | from tornado import gen 72 | from tornado.ioloop import IOLoop 73 | 74 | from tornadohttpclient import TornadoHTTPClient 75 | 76 | # 实例化 77 | http = TornadoHTTPClient() 78 | 79 | @gen.coroutine 80 | def run(): 81 | yield http.post("http://ip.or.domain/login", (("username", "cold"), ("password", "pwd"))) 82 | 83 | IOLoop.instance().run_sync(run) 84 | ``` 85 | 86 | 87 | ## 设置User-Agent 88 | ```python 89 | from tornadohttpclient import TornadoHTTPClient 90 | 91 | http = TornadoHTTPClient() 92 | http.set_user_agent( "Mozilla/5.0 (X11; Linux x86_64)"\ 93 | " AppleWebKit/537.11 (KHTML, like Gecko)"\ 94 | " Chrome/23.0.1271.97 Safari/537.11") 95 | 96 | # 后续的请求都会使用此 User-Agent 头 97 | # ... 98 | ``` 99 | 100 | ### 指定HTTP头 101 | TornadoHTTPClient 的`get`/`post`方法的 `headers`关键字参数可以自定额外的HTTP头信息, 参数类型为一个字典 102 | 103 | 指定User-Agent头 104 | 105 | ```python 106 | from tornado import gen 107 | from tornado.ioloop import IOLoop 108 | 109 | from tornadohttpclient import TornadoHTTPClient 110 | 111 | # 实例化 112 | http = TornadoHTTPClient() 113 | 114 | @gen.coroutine 115 | def run(): 116 | headers = dict((("User-Agent", 117 | "Mozilla/5.0 (X11; Linux x86_64)"\ 118 | " AppleWebKit/537.11 (KHTML, like Gecko)"\ 119 | " Chrome/23.0.1271.97 Safari/537.11"), )) 120 | 121 | yield http.get("http://www.linuxzen.com", headers=headers) 122 | 123 | IOLoop.instance().run_sync(run) 124 | ``` 125 | 126 | ### 使用代理 127 | 128 | TornadoHTTPClient 的`set_proxy`方法可以设置代理, 其接受四个参数, 分别是代理的 主机名/ip 代理的端口 代理用户名 代理用户密码, 如无认证只传前两个即可, `unset_proxy`可以取消代理 129 | ```python 130 | from tornado import gen 131 | from tornado.ioloop import IOLoop 132 | 133 | from tornadohttpclient import TornadoHTTPClient 134 | 135 | # 实例化 136 | http = TornadoHTTPClient() 137 | 138 | @gen.coroutine 139 | def run(): 140 | http.set_proxy("127.0.0.1", 8087) 141 | response = yield http.get("http://shell.appspot.com", callback=callback) 142 | print response.body 143 | http.unset_proxy() 144 | 145 | IOLoop.instance().run_sync(run) 146 | ``` 147 | 148 | ### Cookie 149 | 150 | TornadoHTTPClient会自动记录和装载Cookie, 可以通过 TornadoHTTPClient实例属性 cookie 获取Cookie 151 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Author : cold 5 | # E-mail : wh_linux@126.com 6 | # Date : 13/08/05 16:37:37 7 | # Desc : 8 | # 9 | from __future__ import absolute_import, print_function, division, with_statement 10 | 11 | 12 | import time 13 | import unittest 14 | 15 | from tornado import testing 16 | 17 | from tornadohttpclient import TornadoHTTPClient 18 | 19 | 20 | class TornadoHTTPClientTest(unittest.TestCase): 21 | def setUp(self): 22 | self.http = TornadoHTTPClient() 23 | self.http.debug = True 24 | 25 | def _callback(self, response): 26 | print(response.code) 27 | print("当前链接地址: ") 28 | print(response.effective_url) 29 | self.http.stop() 30 | 31 | def test_get(self): 32 | self.http.get("http://www.linuxzen.com", callback=self._callback) 33 | self.http.start() 34 | 35 | def test_get_args(self): 36 | self.http.get("http://www.baidu.com/s", (("wd", "tornado"),), 37 | callback=self._callback) 38 | 39 | def test_post(self): 40 | params = [("vimcn", u"# 这是TornadoHTTPClient单元测试提交的".encode("utf-8"))] 41 | url = "http://p.vim-cn.com" 42 | def callback(response): 43 | print("打开此链接:", end=" ") 44 | print(response.effective_url) 45 | self.http.stop() 46 | 47 | self.http.post(url, params, callback=callback) 48 | self.http.start() 49 | 50 | def test_head(self): 51 | def callback(response): 52 | pass 53 | 54 | self.http.head("http://linuxzen.com", callback=callback) 55 | 56 | def test_callback_args(self): 57 | def callback(times, response): 58 | print(response.code) 59 | print("当前链接地址: ") 60 | print(response.effective_url) 61 | print("当前请求次数", end=" ") 62 | print(times) 63 | if times == 9: 64 | self.http.stop() 65 | 66 | for i in range(10): 67 | self.http.get("http://www.linuxzen.com", callback=callback, 68 | args=(i,)) 69 | 70 | self.http.start() 71 | 72 | def test_user_agent(self): 73 | user_agent = "Mozilla/5.0 (X11; Linux x86_64)"\ 74 | " AppleWebKit/537.11 (KHTML, like Gecko)"\ 75 | " Chrome/23.0.1271.97 Safari/537.11" 76 | self.http.set_user_agent(user_agent) 77 | 78 | def callback(response): 79 | #self.assertEqual(response.request.headers["User-Agent"], user_agent) 80 | self.http.stop() 81 | 82 | self.http.get("http://www.linuxzen.com", callback=callback) 83 | self.http.start() 84 | 85 | def test_header(self): 86 | headers = {"Origin":"http://www.linuxzen.com"} 87 | def callback(response): 88 | self.assertEqual(response.request.headers["Origin"], 89 | headers.get("Origin")) 90 | self.http.stop() 91 | 92 | self.http.get("http://www.linuxzen.com", callback=callback) 93 | self.http.start() 94 | 95 | def test_cookie(self): 96 | def callback(response): 97 | print("cookie >>>>>>>>>>>>>>>>>>", end=" ") 98 | print(self.http.cookie) 99 | self.http.stop() 100 | 101 | self.http.get("http://www.baidu.com", callback=callback) 102 | self.http.start() 103 | 104 | def test_cookie_jar(self): 105 | def callback(response): 106 | print("cookie jar>>>>>>>>>>>>>>>>>>", end=" ") 107 | print(self.http.cookiejar) 108 | self.http.stop() 109 | 110 | self.http.get("http://www.baidu.com", callback=callback) 111 | self.http.start() 112 | 113 | def test_upload_img(self): 114 | def callback(response): 115 | print("打开图片链接", end = " ") 116 | print(response.effective_url) 117 | self.http.stop() 118 | 119 | self.http.upload("http://dimg.vim-cn.com", "name", "img_test.png", 120 | callback=callback) 121 | self.http.start() 122 | 123 | 124 | class TornadoHTTPClientCoroutineTest(testing.AsyncTestCase): 125 | def setUp(self): 126 | super(TornadoHTTPClientCoroutineTest, self).setUp() 127 | self.http = TornadoHTTPClient() 128 | self.http.debug = True 129 | 130 | def _callback(self, response): 131 | print(response.code) 132 | print("当前链接地址: ") 133 | print(response.effective_url) 134 | self.http.stop() 135 | 136 | @testing.gen_test 137 | def test_get(self): 138 | resp = yield self.http.get("http://www.linuxzen.com") 139 | self._callback(resp) 140 | 141 | @testing.gen_test 142 | def test_get_args(self): 143 | yield self.http.get("http://www.baidu.com/s", (("wd", "tornado"),), 144 | callback=self._callback) 145 | 146 | @testing.gen_test 147 | def test_post(self): 148 | params = [("vimcn", u"# 这是TornadoHTTPClient单元测试提交的".encode("utf-8"))] 149 | url = "http://p.vim-cn.com" 150 | def callback(response): 151 | print("打开此链接:", end=" ") 152 | print(response.effective_url) 153 | 154 | resp = yield self.http.post(url, params) 155 | callback(resp) 156 | 157 | @testing.gen_test 158 | def test_head(self): 159 | resp = yield self.http.head("http://linuxzen.com") 160 | self.assertEqual(resp.code, 200) 161 | 162 | @testing.gen_test 163 | def test_user_agent(self): 164 | user_agent = "Mozilla/5.0 (X11; Linux x86_64)"\ 165 | " AppleWebKit/537.11 (KHTML, like Gecko)"\ 166 | " Chrome/23.0.1271.97 Safari/537.11" 167 | self.http.set_user_agent(user_agent) 168 | 169 | resp = yield self.http.get("http://www.linuxzen.com") 170 | self.assertEqual(resp.request.user_agent, user_agent) 171 | 172 | @testing.gen_test 173 | def test_header(self): 174 | headers = {"Origin":"http://www.linuxzen.com"} 175 | resp = yield self.http.get("http://www.linuxzen.com", headers=headers) 176 | self.assertEqual(resp.request.headers["Origin"], headers["Origin"]) 177 | 178 | @testing.gen_test 179 | def test_cookie(self): 180 | yield self.http.get("http://www.baidu.com") 181 | print(self.http.cookie) 182 | 183 | @testing.gen_test 184 | def test_cookie_jar(self): 185 | yield self.http.get("http://www.baidu.com") 186 | print("cookie jar>>>>>>>>>>>>>>>>>>", end=" ") 187 | print(self.http.cookiejar) 188 | 189 | @testing.gen_test 190 | def test_upload_img(self): 191 | def callback(response): 192 | self.http.stop() 193 | 194 | resp = yield self.http.upload("http://dimg.vim-cn.com", "name", 195 | "img_test.png") 196 | print("打开图片链接", end = " ") 197 | print(resp.effective_url) 198 | 199 | def test_set_proxy(self): 200 | self.http.set_proxy('127.0.0.1') 201 | 202 | 203 | def main(): 204 | loader = unittest.TestLoader() 205 | suite = loader.loadTestsFromNames(["tests.TornadoHTTPClientTest", 206 | "tests.TornadoHTTPClientCoroutineTest"]) 207 | runner = unittest.TextTestRunner(verbosity = 2) 208 | runner.run(suite) 209 | 210 | if __name__ == "__main__": 211 | main() 212 | 213 | -------------------------------------------------------------------------------- /tornadohttpclient.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # 4 | # Author : cold 5 | # E-mail : wh_linux@126.com 6 | # Date : 13/08/05 10:49:47 7 | # Desc : 8 | # 9 | from __future__ import absolute_import, print_function, division 10 | from __future__ import with_statement 11 | 12 | import os 13 | import json 14 | import mimetypes 15 | import itertools 16 | 17 | from functools import partial 18 | 19 | try: 20 | from mimetools import choose_boundary 21 | except ImportError: 22 | from email.generator import _make_boundary as choose_boundary 23 | 24 | try: 25 | from cookielib import Cookie, CookieJar 26 | except ImportError: 27 | from http.cookiejar import Cookie, CookieJar # py3 28 | 29 | try: 30 | from urllib import urlencode 31 | except ImportError: 32 | from urllib.parse import urlencode 33 | 34 | import pycurl 35 | 36 | from tornado import httputil 37 | from tornado import httpclient 38 | from tornado import escape 39 | from tornado import util 40 | 41 | try: 42 | from tornado.curl_httpclient import CurlAsyncHTTPClient 43 | except ImportError: 44 | CurlAsyncHTTPClient = httpclient.AsyncHTTPClient 45 | 46 | from tornado.ioloop import IOLoop 47 | 48 | 49 | class TornadoHTTPClient(CurlAsyncHTTPClient): 50 | """ 封装 tornado.curl_httpclient.CurlAsyncHTTPClient 支持 cookie 并封装的 51 | 更加易用是指支持如下方法: 52 | get/post/put/patch/head/delete/options 53 | :: 54 | from tornado import gen 55 | from tornado.web import RequestHandler 56 | from tornadohttpclient import TornadoHTTPClient 57 | class DemoHandler(RequestHandler): 58 | @gen.coroutine 59 | def get(self): 60 | http_client = TornadoHTTPClient() 61 | response = yield http_client.get("http://www.google.com", 62 | {"s": "word"}) 63 | self.write(response.body) 64 | """ 65 | 66 | def initialize(self, *args, **kwargs): 67 | super(TornadoHTTPClient, self).initialize(*args, **kwargs) 68 | self._cookie = {} 69 | self._proxy = {} 70 | self._user_agent = None 71 | self.keep_alive = True 72 | self.use_cookie = True 73 | self.debug = False 74 | self.validate_cert = True 75 | self._headers = {} 76 | 77 | self._share = None 78 | if pycurl is not None: 79 | self._share = pycurl.CurlShare() 80 | self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE) 81 | self._share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS) 82 | for curl in self._curls: 83 | self._setup_curl(curl) 84 | 85 | def _setup_curl(self, curl): 86 | curl.setopt(pycurl.SHARE, self._share) 87 | 88 | def set_user_agent(self, user_agent): 89 | self._user_agent = user_agent 90 | 91 | def set_global_headers(self, headers): 92 | self._headers = headers 93 | 94 | def set_proxy(self, host, port=8080, username=None, password=None): 95 | assert isinstance(port, int) 96 | self._proxy["proxy_host"] = host 97 | self._proxy["proxy_port"] = port 98 | if username: 99 | self._proxy["proxy_username"] = username 100 | 101 | if password: 102 | self._proxy["proxy_password"] = password 103 | 104 | def unset_proxy(self): 105 | self._proxy = {} 106 | 107 | def wrap_prepare_curl_callback(self, callback): 108 | def _wrap_prepare_curl_callback(curl): 109 | if self.debug: 110 | curl.setopt(pycurl.VERBOSE, 1) 111 | 112 | if callback: 113 | return callback(curl) 114 | return _wrap_prepare_curl_callback 115 | 116 | def wrap_callback(self, callback, args=(), kwargs={}): 117 | return partial(callback, *args, **kwargs) 118 | 119 | def get_url(self, url, params): 120 | """ 拼接 URL 121 | :param url: url 122 | :param params: URL 参数 123 | """ 124 | if params is not None: 125 | if isinstance(params, (list, dict, tuple)): 126 | params = urlencode(params) 127 | 128 | if "?" not in url: 129 | url += "?" + params 130 | else: 131 | url += "&" + params 132 | return url.rstrip("?") 133 | 134 | def get_urlencoded_body(self, data): 135 | result = [] 136 | data = data.items() if isinstance(data, dict) else data 137 | 138 | for key, val in data: 139 | if val is not None: 140 | result.append((key, val)) 141 | 142 | return urlencode(result) 143 | 144 | def get_json_body(self, data): 145 | return json.dumps(data) 146 | 147 | def get_multipart_body(self, data): 148 | raise NotImplementedError() 149 | 150 | def get_request_body(self, content_type, data): 151 | """ 根据请求内容的类型来获取请求包体 """ 152 | if isinstance(data, util.basestring_type): 153 | return escape.utf8(data) 154 | 155 | if content_type is None: 156 | return self.get_urlencoded_body(data) 157 | 158 | if content_type[:16] == "application/json": 159 | return self.get_json_body(data) 160 | elif content_type[:19] == "multipart/form-data": 161 | return self.get_multipart_body(data) 162 | else: 163 | return self.get_urlencoded_body(data) 164 | 165 | def make_request(self, url, data=None, **kwargs): 166 | """ 构造 HTTP 请求 167 | :param url: 请求路径 168 | :param data: 请求参数 169 | :param url_params: 放在 URL 路径上的请求参数, 用于请求方法非 GET 但 170 | 有参数放在 URL 上 171 | :param **kwargs: 传递给 tornado.httpclient.HTTPRequest 的其他参数 172 | """ 173 | headers = httputil.HTTPHeaders() 174 | headers.update(kwargs.get("headers", {})) 175 | headers.update(self._headers) 176 | self.keep_alive and headers.update(Connection="keep-alive") 177 | kwargs.update(validate_cert=self.validate_cert) 178 | 179 | kwargs["headers"] = headers 180 | 181 | method = kwargs.get("method", "GET").upper() 182 | if method in ["GET", "HEAD", "DELETE"]: 183 | url_params = data 184 | else: 185 | url_params = kwargs.pop("url_params", {}) 186 | if not kwargs.get('body'): 187 | kwargs["body"] = self.get_request_body( 188 | headers.get("content-type"), data) 189 | 190 | curl_callback = kwargs.pop("prepare_curl_callback", None) 191 | curl_callback = self.wrap_prepare_curl_callback(curl_callback) 192 | 193 | kwargs.update(self._proxy) 194 | self._user_agent and kwargs.update(user_agent=self._user_agent) 195 | kwargs.update(prepare_curl_callback=curl_callback) 196 | 197 | url = self.get_url(url, url_params) 198 | return httpclient.HTTPRequest(url, **kwargs) 199 | 200 | def request(self, url, data=None, **kwargs): 201 | """ 封装 Tornado 异步 HTTP 请求, 并记录日志 """ 202 | callback = kwargs.pop("callback", None) 203 | args, kw = kwargs.pop("args", ()), kwargs.pop("kwargs", {}) 204 | 205 | if callable(callback): 206 | callback = self.wrap_callback(callback, args, kw) 207 | 208 | request = self.make_request(url, data, **kwargs) 209 | return self.fetch(request, callback=callback) 210 | 211 | def get(self, url, data=None, **kwargs): 212 | return self.request(url, data, **kwargs) 213 | 214 | def post(self, url, data, **kwargs): 215 | return self.request(url, data, method="POST", **kwargs) 216 | 217 | def put(self, url, data, **kwargs): 218 | return self.request(url, data, method="PUT", **kwargs) 219 | 220 | def head(self, url, data=None, **kwargs): 221 | return self.request(url, data, method="HEAD", **kwargs) 222 | 223 | def delete(self, url, data=None, **kwargs): 224 | return self.request(url, data, method="DELETE", **kwargs) 225 | 226 | def options(self, url, data, **kwargs): 227 | return self.request(url, data, method="OPTIONS", **kwargs) 228 | 229 | def patch(self, url, data, **kwargs): 230 | return self.request(url, data, method="PATCH", **kwargs) 231 | 232 | def close(self): 233 | super(TornadoHTTPClient, self).close() 234 | self._force_timeout_callback.callback = None 235 | self._multi = None 236 | 237 | def upload(self, url, field, path, params={}, mimetype=None, 238 | callback=None, **kwargs): 239 | method = kwargs.pop("method", "POST") 240 | form = UploadForm() 241 | [form.add_field(name, value) for name, value in params.items()] 242 | _, fname = os.path.split(path) 243 | form.add_file(field, fname, open(path, 'rb'), mimetype) 244 | kwargs.update(body=str(form)) 245 | kwargs.update(method=method) 246 | kwargs.update(headers={"Content-Type": form.get_content_type()}) 247 | kwargs["callback"] = callback 248 | return self.request(url, **kwargs) 249 | 250 | @property 251 | def cookie(self): 252 | lst = [] 253 | for curl in self._curls: 254 | lst.extend(curl.getinfo(pycurl.INFO_COOKIELIST)) 255 | 256 | return self._parse_cookie(lst) 257 | 258 | def _parse_cookie(self, lst): 259 | for item in lst: 260 | domain, domain_specified, path, path_specified, expires,\ 261 | name, value = item.split("\t") 262 | 263 | cookie = Cookie(0, name, value, None, False, domain, 264 | domain_specified.lower() == "true", 265 | domain.startswith("."), path, 266 | path_specified.lower() == "true", False, expires, 267 | False, None, None, {}) 268 | 269 | self._cookie.setdefault(domain, {}) 270 | self._cookie[domain].setdefault(path, {}) 271 | self._cookie[domain][path].update({name: cookie}) 272 | 273 | return self._cookie 274 | 275 | @property 276 | def cookiejar(self): 277 | cookiejar = CookieJar() 278 | for domain, items in self.cookie.items(): 279 | for path, names in items.items(): 280 | for name, cookie in names.items(): 281 | cookiejar.set_cookie(cookie) 282 | 283 | return cookiejar 284 | 285 | def start(self): 286 | IOLoop.instance().start() 287 | 288 | def stop(self): 289 | IOLoop.instance().stop() 290 | 291 | 292 | class UploadForm(object): 293 | def __init__(self): 294 | self.form_fields = [] 295 | self.files = [] 296 | self.boundary = choose_boundary() 297 | self.content_type = 'multipart/form-data; boundary=%s' % self.boundary 298 | return 299 | 300 | def get_content_type(self): 301 | return self.content_type 302 | 303 | def add_field(self, name, value): 304 | self.form_fields.append((str(name), str(value))) 305 | return 306 | 307 | def add_file(self, fieldname, filename, fileHandle, mimetype=None): 308 | # use str convert to str to compatible with py35 309 | body = str(fileHandle.read()) 310 | if mimetype is None: 311 | mimetype = (mimetypes.guess_type(filename)[0] or 312 | 'applicatioin/octet-stream') 313 | self.files.append((fieldname, filename, mimetype, body)) 314 | return 315 | 316 | def __str__(self): 317 | parts = [] 318 | part_boundary = '--' + self.boundary 319 | 320 | parts.extend( 321 | [ 322 | part_boundary, 323 | 'Content-Disposition: form-data; name="%s"' % name, 324 | '', 325 | value, 326 | ] for name, value in self.form_fields) 327 | if self.files: 328 | parts.extend([ 329 | part_boundary, 330 | 'Content-Disposition: form-data; name="%s"; filename="%s"' % ( 331 | field_name, filename), 332 | 'Content-Type: %s' % content_type, 333 | '', 334 | body, 335 | ] for field_name, filename, content_type, body in self.files) 336 | 337 | flattened = list(itertools.chain(*parts)) 338 | flattened.append('--' + self.boundary + '--') 339 | flattened.append('') 340 | return '\r\n'.join(flattened) 341 | --------------------------------------------------------------------------------