├── README.md └── hack_requests.py /README.md: -------------------------------------------------------------------------------- 1 | # HackRequests: 2 | It is a dedicated requests lib that supports cookie, headers, get/post, etc. And it also supports rendering the response (e.g. Javascript, CSS, etc.) of GET requests by using PhantomJs enginee. 3 | 4 | # Requirements: 5 | ## Install Selenium: 6 | * sudo pip install -U selenium 7 | * sudo apt-get install libfontconfig 8 | 9 | ## Install PhantomJs: 10 | * For Linux 64-bit: 11 | * wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 12 | * tar -jxvf phantomjs-2.1.1-linux-x86_64.tar.bz2 13 | 14 | * For Linux 32-bit: 15 | * wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-i686.tar.bz2 16 | * tar -jxvf phantomjs-2.1.1-linux-i686.tar.bz2 17 | 18 | # Configuration: 19 | Modify `self.executable_path` to the path of PhantomJS binary, e.g. `self.executable_path='/home/ubuntu/phantomjs-2.1.1-linux-x86_64/bin/phantomjs'` 20 | 21 | # Usage: 22 | ``` 23 | import requests 24 | import re 25 | import time 26 | from hack_requests import HackRequests 27 | 28 | request_info = {} # It is a dict which contains all HTTP headers and data, the format is similar as Burp request. 29 | request_info['protocol'] = 'http' 30 | request_info['host'] = 'example.com' 31 | request_info['port'] = 80 32 | request_info['path'] = '/iam/the/url?para=PARA' 33 | request_info['user_agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:45.0) Gecko/20100101 Firefox/45.0' 34 | request_info['accept'] = '*/*' 35 | request_info['accept_language'] = 'en-US,en;q=0.5' 36 | request_info['accept_encoding'] = 'gzip, deflate' 37 | request_info['referer'] = "http://example.com/" 38 | request_info['cookie'] = "PHPSESSID=look_at_me_i_am_the_sessionid; Second_Cookie=i_am_second_cookie" 39 | request_info['post_data'] = "para1=PARA1¶2=PARA2" 40 | 41 | # Send GET request by using PhantomJS 42 | r_get_p = HackRequests(request_info, 'PHANTOMJS').get_request() 43 | 44 | # Send GET request by using Requests 45 | r_get_r = HackRequests(request_info).get_request() 46 | 47 | # Send POST request by using Requests 48 | r_post_r = HackRequests(request_info).post_request() 49 | ``` 50 | -------------------------------------------------------------------------------- /hack_requests.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ Hack Requests 4 | 5 | This is a dedicated requests lib that supports cookie, headers, get/post, etc. And it also supports rendering the response (e.g. Javascript, CSS, etc.) of GET requests by using PhantomJs enginee. 6 | 7 | """ 8 | 9 | import requests 10 | import json 11 | from selenium import webdriver 12 | 13 | 14 | class HackRequests(object): 15 | def __init__(self, request_info, lib = 'REQUESTS'): 16 | self.request_info = request_info 17 | self.lib = lib 18 | self.url = "{}://{}:{}{}".format(self.request_info['protocol'], self.request_info['host'], self.request_info['port'], self.request_info['path']) 19 | self.TIME_OUT = 10 # timeout of requests 20 | 21 | # configurations for PhantomJS 22 | self.headers = {'User-Agent': self.request_info['user_agent'], 23 | 'Accept': self.request_info['accept'], 24 | 'Accept-Language': self.request_info['accept_language'], 25 | 'Accept-Encoding': self.request_info['accept_encoding'], 26 | 'Cookie': self.request_info['cookie'], 27 | 'Referer': self.request_info['referer']} 28 | self.executable_path='' 29 | self.service_args=[] 30 | self.service_args.append('--load-images=no') 31 | self.service_args.append('--local-url-access=no') # Don't allow to load local file 32 | self.service_args.append('--disk-cache=yes') 33 | self.service_args.append('--ignore-ssl-errors=true') 34 | self.driver = self.init_phantomjs_driver(executable_path = self.executable_path, service_args = self.service_args) 35 | 36 | def get_request(self): 37 | if self.lib == "REQUESTS": 38 | headers = {'User-Agent': self.request_info['user_agent'], 39 | 'Accept': self.request_info['accept'], 40 | 'Accept-Language': self.request_info['accept_language'], 41 | 'Accept-Encoding': self.request_info['accept_encoding'], 42 | 'Referer': self.request_info['referer']} 43 | try: 44 | if self.request_info['cookie']: 45 | cookies = self.get_cookie() 46 | r = requests.get(self.url, headers = headers, cookies = cookies, timeout = self.TIME_OUT, verify=False).content 47 | else: 48 | r = requests.get(self.url, headers = headers, timeout = self.TIME_OUT, verify=False).content 49 | except Exception, e: 50 | print str(e) 51 | r = "" 52 | return r 53 | elif self.lib == "PHANTOMJS": 54 | try: 55 | self.driver.get(self.url) 56 | r = self.driver.page_source 57 | except Exception, e: 58 | print str(e) 59 | r = "" 60 | return r 61 | 62 | def post_request(self): 63 | data = self.get_post_data() 64 | headers = {'User-Agent': self.request_info['user_agent'], 65 | 'Accept': self.request_info['accept'], 66 | 'Accept-Language': self.request_info['accept_language'], 67 | 'Accept-Encoding': self.request_info['accept_encoding'], 68 | 'Referer': self.request_info['referer']} 69 | try: 70 | if self.request_info['cookie']: 71 | cookies = self.get_cookie() 72 | r = requests.post(self.url, data = data, headers = headers, cookies = cookies, timeout = self.TIME_OUT, verify=False).content 73 | else: 74 | r = requests.post(self.url, data = data, headers = headers, timeout = self.TIME_OUT, verify=False).content 75 | except Exception, e: 76 | print str(e) 77 | r = "" 78 | return r 79 | 80 | def get_cookie(self): 81 | cookies = {} 82 | for line in self.request_info['cookie'].split(';'): 83 | name, value = line.strip().split('=', 1) # 1 means only split one time 84 | cookies[name] = value 85 | 86 | return cookies 87 | 88 | def get_post_data(self): 89 | if not self.request_info['content_type']: 90 | post_data = "" 91 | elif "application/x-www-form-urlencoded" in self.request_info['content_type']: 92 | post_data = {} 93 | for line in self.request_info['post_data'].split('&'): 94 | name, value = line.strip().split('=', 1) # 1 means only split one time 95 | post_data[name] = value 96 | elif "application/json" in self.request_info['content_type']: 97 | post_data = json.dumps(self.request_info['post_data']) 98 | else: 99 | post_data = "" 100 | 101 | return post_data 102 | 103 | def init_phantomjs_driver(self, *args, **kwargs): 104 | for key, value in self.headers.iteritems(): 105 | webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.{}'.format(key)] = value 106 | driver = webdriver.PhantomJS(*args, **kwargs) 107 | driver.set_page_load_timeout(self.TIME_OUT) 108 | return driver 109 | --------------------------------------------------------------------------------