├── CHANGELOG.txt ├── example_groups.php ├── example.php ├── RollingCurlGroup.php ├── README.txt └── RollingCurl.php /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | Rolling Curl changelog 2 | ====================== 3 | 4 | September 13, 2010 5 | ------------------ 6 | - Bug #12, #14: Fixed default options overriding (LionsAd) 7 | - Bug #10: Added use of curl_multi_select to avoid burning CPU (LionsAd) 8 | - Enh #6, #9: Added $request as parameter to callback function (LionsAd) 9 | - Chg: Request renamed to RollingCurlRequest (LionsAd) 10 | - Added RollingCurlGroup class that allows processing groups of requests (LionsAd) 11 | - More cleanup at unsetting a class (LionsAd) 12 | - Timeout parameter for curl_multi_select is now configurable (LionsAd) 13 | - single_curl now returns true (LionsAd) 14 | - Readme corrections (Alexander Makarov) 15 | - Code cleanup (Alexander Makarov) -------------------------------------------------------------------------------- /example_groups.php: -------------------------------------------------------------------------------- 1 | url . "\n"; 10 | if ($this->test_verbose) 11 | print_r($info); 12 | 13 | parent::process($output, $info); 14 | } 15 | } 16 | 17 | class TestCurlGroup extends RollingCurlGroup { 18 | function process($output, $info, $request) { 19 | echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n"; 20 | parent::process($output, $info, $request); 21 | } 22 | 23 | function finished() { 24 | echo "Group CB: Finished" . $this->name . "\n"; 25 | parent::finished(); 26 | } 27 | } 28 | 29 | $group = new TestCurlGroup("High"); 30 | $group->add(new TestCurlRequest("www.google.de")); 31 | $group->add(new TestCurlRequest("www.yahoo.de")); 32 | $group->add(new TestCurlRequest("www.newyorktimes.com")); 33 | $reqs[] = $group; 34 | 35 | $group = new TestCurlGroup("Normal"); 36 | $group->add(new TestCurlRequest("twitter.com")); 37 | $group->add(new TestCurlRequest("www.bing.com")); 38 | $group->add(new TestCurlRequest("m.facebook.com")); 39 | $reqs[] = $group; 40 | 41 | $reqs[] = new TestCurlRequest("www.kernel.org"); 42 | 43 | // No callback here, as its done in Request class 44 | $rc = new GroupRollingCurl(); 45 | 46 | foreach ($reqs as $req) 47 | $rc->add($req); 48 | 49 | $rc->execute(); -------------------------------------------------------------------------------- /example.php: -------------------------------------------------------------------------------- 1 | (.*?)~i", $response, $out)) { 17 | $title = $out[1]; 18 | } 19 | echo "$title
"; 20 | print_r($info); 21 | print_r($request); 22 | echo "
"; 23 | } 24 | 25 | // single curl request 26 | $rc = new RollingCurl("request_callback"); 27 | $rc->request("http://www.msn.com"); 28 | $rc->execute(); 29 | 30 | // another single curl request 31 | $rc = new RollingCurl("request_callback"); 32 | $rc->request("http://www.google.com"); 33 | $rc->execute(); 34 | 35 | echo "
"; 36 | 37 | // top 20 sites according to alexa (11/5/09) 38 | $urls = array("http://www.google.com", 39 | "http://www.facebook.com", 40 | "http://www.yahoo.com", 41 | "http://www.youtube.com", 42 | "http://www.live.com", 43 | "http://www.wikipedia.com", 44 | "http://www.blogger.com", 45 | "http://www.msn.com", 46 | "http://www.baidu.com", 47 | "http://www.yahoo.co.jp", 48 | "http://www.myspace.com", 49 | "http://www.qq.com", 50 | "http://www.google.co.in", 51 | "http://www.twitter.com", 52 | "http://www.google.de", 53 | "http://www.microsoft.com", 54 | "http://www.google.cn", 55 | "http://www.sina.com.cn", 56 | "http://www.wordpress.com", 57 | "http://www.google.co.uk"); 58 | 59 | $rc = new RollingCurl("request_callback"); 60 | $rc->window_size = 20; 61 | foreach ($urls as $url) { 62 | $request = new RollingCurlRequest($url); 63 | $rc->add($request); 64 | } 65 | $rc->execute(); 66 | -------------------------------------------------------------------------------- /RollingCurlGroup.php: -------------------------------------------------------------------------------- 1 | group = $group; 28 | } 29 | 30 | /** 31 | * Process the request 32 | * 33 | * 34 | */ 35 | function process($output, $info) { 36 | if ($this->group) 37 | $this->group->process($output, $info, $this); 38 | } 39 | 40 | /** 41 | * @return void 42 | */ 43 | public function __destruct() { 44 | unset($this->group); 45 | parent::__destruct(); 46 | } 47 | 48 | } 49 | 50 | /** 51 | * A group of curl requests. 52 | * 53 | * @throws RollingCurlGroupException * 54 | */ 55 | class RollingCurlGroup { 56 | /** 57 | * @var string group name 58 | */ 59 | protected $name; 60 | 61 | /** 62 | * @var int total number of requests in a group 63 | */ 64 | protected $num_requests = 0; 65 | 66 | /** 67 | * @var int total number of finished requests in a group 68 | */ 69 | protected $finished_requests = 0; 70 | 71 | /** 72 | * @var array requests array 73 | */ 74 | private $requests = array(); 75 | 76 | /** 77 | * @param string $name group name 78 | * @return void 79 | */ 80 | function __construct($name) { 81 | $this->name = $name; 82 | } 83 | 84 | /** 85 | * @return void 86 | */ 87 | public function __destruct() { 88 | unset($this->name, $this->num_requests, $this->finished_requests, $this->requests); 89 | } 90 | 91 | /** 92 | * Adds request to a group 93 | * 94 | * @throws RollingCurlGroupException 95 | * @param RollingCurlGroupRequest|array $request 96 | * @return bool 97 | */ 98 | function add($request) { 99 | if ($request instanceof RollingCurlGroupRequest) { 100 | $request->setGroup($this); 101 | $this->num_requests++; 102 | $this->requests[] = $request; 103 | } 104 | else if (is_array($request)) { 105 | foreach ($request as $req) 106 | $this->add($req); 107 | } 108 | else 109 | throw new RollingCurlGroupException("add: Request needs to be of instance RollingCurlGroupRequest"); 110 | 111 | return true; 112 | } 113 | 114 | /** 115 | * @throws RollingCurlGroupException 116 | * @param RollingCurl $rc 117 | * @return bool 118 | */ 119 | function addToRC(RollingCurl $rc){ 120 | $ret = true; 121 | 122 | while (count($this->requests) > 0){ 123 | $ret1 = $rc->add(array_shift($this->requests)); 124 | if (!$ret1) 125 | $ret = false; 126 | } 127 | 128 | return $ret; 129 | } 130 | 131 | /** 132 | * Override to implement custom response processing. 133 | * 134 | * Don't forget to call parent::process(). 135 | * 136 | * @param string $output received page body 137 | * @param array $info holds various information about response such as HTTP response code, content type, time taken to make request etc. 138 | * @param RollingCurlRequest $request request used 139 | * @return void 140 | */ 141 | function process($output, $info, $request) { 142 | $this->finished_requests++; 143 | 144 | if ($this->finished_requests >= $this->num_requests) 145 | $this->finished(); 146 | } 147 | 148 | /** 149 | * Override to execute code after all requests in a group are processed. 150 | * 151 | * @return void 152 | */ 153 | function finished() { 154 | } 155 | 156 | } 157 | 158 | /** 159 | * Group version of rolling curl 160 | */ 161 | class GroupRollingCurl extends RollingCurl { 162 | 163 | /** 164 | * @var mixed common callback for all groups 165 | */ 166 | private $group_callback = null; 167 | 168 | /** 169 | * @param string $output received page body 170 | * @param array $info holds various information about response such as HTTP response code, content type, time taken to make request etc. 171 | * @param RollingCurlRequest $request request used 172 | * @return void 173 | */ 174 | protected function process($output, $info, $request) { 175 | if ($request instanceof RollingCurlGroupRequest) 176 | $request->process($output, $info); 177 | 178 | if (is_callable($this->group_callback)) 179 | call_user_func($this->group_callback, $output, $info, $request); 180 | } 181 | 182 | /** 183 | * @param mixed $callback common callback for all groups 184 | * @return void 185 | */ 186 | function __construct($callback = null) { 187 | $this->group_callback = $callback; 188 | 189 | parent::__construct(array(&$this, "process")); 190 | } 191 | 192 | /** 193 | * Adds a group to processing queue 194 | * 195 | * @param RollingCurlGroup|Request $request 196 | * @return bool 197 | */ 198 | public function add($request) { 199 | if ($request instanceof RollingCurlGroup) 200 | return $request->addToRC($this); 201 | else 202 | return parent::add($request); 203 | } 204 | 205 | /** 206 | * Execute processing 207 | * 208 | * @param int $window_size Max number of simultaneous connections 209 | * @return bool|string 210 | */ 211 | public function execute($window_size = null) { 212 | if (count($this->requests) == 0) 213 | return false; 214 | 215 | return parent::execute($window_size); 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | Rolling Curl 2 | ============ 3 | 4 | RollingCurl allows you to process multiple HTTP requests in parallel using CURL PHP library. 5 | 6 | Released under the Apache License 2.0. 7 | 8 | Authors 9 | ------- 10 | - Was originally written by [Josh Fraser](joshfraser.com). 11 | - Currently maintained by [Alexander Makarov](http://rmcreative.ru/). 12 | - Received significant updates and patched from [LionsAd](http://github.com/LionsAd/rolling-curl). 13 | 14 | Overview 15 | -------- 16 | RollingCurl is a more efficient implementation of curl_multi() curl_multi is a great way to process multiple HTTP requests in parallel in PHP. 17 | curl_multi is particularly handy when working with large data sets (like fetching thousands of RSS feeds at one time). Unfortunately there is 18 | very little documentation on the best way to implement curl_multi. As a result, most of the examples around the web are either inefficient or 19 | fail entirely when asked to handle more than a few hundred requests. 20 | 21 | The problem is that most implementations of curl_multi wait for each set of requests to complete before processing them. If there are too many requests 22 | to process at once, they usually get broken into groups that are then processed one at a time. The problem with this is that each group has to wait for 23 | the slowest request to download. In a group of 100 requests, all it takes is one slow one to delay the processing of 99 others. The larger the number of 24 | requests you are dealing with, the more noticeable this latency becomes. 25 | 26 | The solution is to process each request as soon as it completes. This eliminates the wasted CPU cycles from busy waiting. Also there is a queue of 27 | cURL requests to allow for maximum throughput. Each time a request is completed, a new one is added from the queue. By dynamically adding and removing 28 | links, we keep a constant number of links downloading at all times. This gives us a way to throttle the amount of simultaneous requests we are sending. 29 | The result is a faster and more efficient way of processing large quantities of cURL requests in parallel. 30 | 31 | Callbacks 32 | --------- 33 | 34 | Each of requests usually do have a callback to process results that is being executed when request is done 35 | (both successfully or not). 36 | 37 | Callback accepts three parameters and can look like the following one: 38 | ~~~ 39 | [php] 40 | function request_callback($response, $info, $request){ 41 | // doing something with the data received 42 | } 43 | ~~~ 44 | 45 | - $response contains received page body. 46 | - $info is an associative array that holds various information about response such as HTTP response code, content type, 47 | time taken to make request etc. 48 | - $request contains RollingCurlRequest that was used to make request. 49 | 50 | Examples 51 | -------- 52 | ### Hello world 53 | 54 | ~~~ 55 | [php] 56 | // an array of URL's to fetch 57 | $urls = array("http://www.google.com", 58 | "http://www.facebook.com", 59 | "http://www.yahoo.com"); 60 | 61 | // a function that will process the returned responses 62 | function request_callback($response, $info, $request) { 63 | // parse the page title out of the returned HTML 64 | if (preg_match("~(.*?)~i", $response, $out)) { 65 | $title = $out[1]; 66 | } 67 | echo "$title
"; 68 | print_r($info); 69 | echo "
"; 70 | } 71 | 72 | // create a new RollingCurl object and pass it the name of your custom callback function 73 | $rc = new RollingCurl("request_callback"); 74 | // the window size determines how many simultaneous requests to allow. 75 | $rc->window_size = 20; 76 | foreach ($urls as $url) { 77 | // add each request to the RollingCurl object 78 | $request = new RollingCurlRequest($url); 79 | $rc->add($request); 80 | } 81 | $rc->execute(); 82 | ~~~ 83 | 84 | 85 | ### Setting custom options 86 | 87 | Set custom options for EVERY request: 88 | 89 | ~~~ 90 | [php] 91 | $rc = new RollingCurl("request_callback"); 92 | $rc->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true); 93 | $rc->execute(); 94 | ~~~ 95 | 96 | Set custom options for A SINGLE request: 97 | 98 | ~~~ 99 | [php] 100 | $rc = new RollingCurl("request_callback"); 101 | $request = new RollingCurlRequest($url); 102 | $request->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true); 103 | $rc->add($request); 104 | $rc->execute(); 105 | ~~~ 106 | 107 | ### Shortcuts 108 | 109 | ~~~ 110 | [php] 111 | $rc = new RollingCurl("request_callback"); 112 | $rc->get("http://www.google.com"); 113 | $rc->get("http://www.yahoo.com"); 114 | $rc->execute(); 115 | ~~~ 116 | 117 | ### Class callbacks 118 | 119 | ~~~ 120 | [php] 121 | class MyInfoCollector { 122 | private $rc; 123 | 124 | function __construct(){ 125 | $this->rc = new RollingCurl(array($this, 'processPage')); 126 | } 127 | 128 | function processPage($response, $info, $request){ 129 | //... 130 | } 131 | 132 | function run($urls){ 133 | foreach ($urls as $url){ 134 | $request = new RollingCurlRequest($url); 135 | $this->rc->add($request); 136 | } 137 | $this->rc->execute(); 138 | } 139 | } 140 | 141 | $collector = new MyInfoCollector(); 142 | $collector->run(array( 143 | 'http://google.com/', 144 | 'http://yahoo.com/' 145 | )); 146 | ~~~ 147 | 148 | ### Using RollingCurlGroup 149 | 150 | ~~~ 151 | [php] 152 | class TestCurlRequest extends RollingCurlGroupRequest { 153 | public $test_verbose = true; 154 | 155 | function process($output, $info) { 156 | echo "Processing " . $this->url . "\n"; 157 | if ($this->test_verbose) 158 | print_r($info); 159 | 160 | parent::process($output, $info); 161 | } 162 | } 163 | 164 | class TestCurlGroup extends RollingCurlGroup { 165 | function process($output, $info, $request) { 166 | echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n"; 167 | parent::process($output, $info, $request); 168 | } 169 | 170 | function finished() { 171 | echo "Group CB: Finished" . $this->name . "\n"; 172 | parent::finished(); 173 | } 174 | } 175 | 176 | $group = new TestCurlGroup("High"); 177 | $group->add(new TestCurlRequest("www.google.de")); 178 | $group->add(new TestCurlRequest("www.yahoo.de")); 179 | $group->add(new TestCurlRequest("www.newyorktimes.com")); 180 | $reqs[] = $group; 181 | 182 | $group = new TestCurlGroup("Normal"); 183 | $group->add(new TestCurlRequest("twitter.com")); 184 | $group->add(new TestCurlRequest("www.bing.com")); 185 | $group->add(new TestCurlRequest("m.facebook.com")); 186 | $reqs[] = $group; 187 | 188 | $reqs[] = new TestCurlRequest("www.kernel.org"); 189 | 190 | // No callback here, as its done in Request class 191 | $rc = new GroupRollingCurl(); 192 | 193 | foreach ($reqs as $req) 194 | $rc->add($req); 195 | 196 | $rc->execute(); 197 | ~~~ 198 | 199 | The same function (add) can be used both for adding requests and groups of requests. 200 | The "callback" in request and groups is: 201 | 202 | process($output, $info) 203 | 204 | and 205 | 206 | process($output, $info, $request) 207 | 208 | Also you can override RollingCurlGroup::finished() that will be executed right after finishing group processing. 209 | 210 | $Id$ -------------------------------------------------------------------------------- /RollingCurl.php: -------------------------------------------------------------------------------- 1 | url = $url; 31 | $this->method = $method; 32 | $this->post_data = $post_data; 33 | $this->headers = $headers; 34 | $this->options = $options; 35 | } 36 | 37 | /** 38 | * @return void 39 | */ 40 | public function __destruct() { 41 | unset($this->url, $this->method, $this->post_data, $this->headers, $this->options); 42 | } 43 | } 44 | 45 | /** 46 | * RollingCurl custom exception 47 | */ 48 | class RollingCurlException extends Exception { 49 | } 50 | 51 | /** 52 | * Class that holds a rolling queue of curl requests. 53 | * 54 | * @throws RollingCurlException 55 | */ 56 | class RollingCurl { 57 | /** 58 | * @var int 59 | * 60 | * Window size is the max number of simultaneous connections allowed. 61 | * 62 | * REMEMBER TO RESPECT THE SERVERS: 63 | * Sending too many requests at one time can easily be perceived 64 | * as a DOS attack. Increase this window_size if you are making requests 65 | * to multiple servers or have permission from the receving server admins. 66 | */ 67 | private $window_size = 5; 68 | 69 | /** 70 | * @var float 71 | * 72 | * Timeout is the timeout used for curl_multi_select. 73 | */ 74 | private $timeout = 10; 75 | 76 | /** 77 | * @var string|array 78 | * 79 | * Callback function to be applied to each result. 80 | */ 81 | private $callback; 82 | 83 | /** 84 | * @var array 85 | * 86 | * Set your base options that you want to be used with EVERY request. 87 | */ 88 | protected $options = array( 89 | CURLOPT_SSL_VERIFYPEER => 0, 90 | CURLOPT_RETURNTRANSFER => 1, 91 | CURLOPT_CONNECTTIMEOUT => 30, 92 | CURLOPT_TIMEOUT => 30 93 | ); 94 | 95 | /** 96 | * @var array 97 | */ 98 | private $headers = array(); 99 | 100 | /** 101 | * @var Request[] 102 | * 103 | * The request queue 104 | */ 105 | private $requests = array(); 106 | 107 | /** 108 | * @var RequestMap[] 109 | * 110 | * Maps handles to request indexes 111 | */ 112 | private $requestMap = array(); 113 | 114 | /** 115 | * @param $callback 116 | * Callback function to be applied to each result. 117 | * 118 | * Can be specified as 'my_callback_function' 119 | * or array($object, 'my_callback_method'). 120 | * 121 | * Function should take three parameters: $response, $info, $request. 122 | * $response is response body, $info is additional curl info. 123 | * $request is the original request 124 | * 125 | * @return void 126 | */ 127 | function __construct($callback = null) { 128 | $this->callback = $callback; 129 | } 130 | 131 | /** 132 | * @param string $name 133 | * @return mixed 134 | */ 135 | public function __get($name) { 136 | return (isset($this->{$name})) ? $this->{$name} : null; 137 | } 138 | 139 | /** 140 | * @param string $name 141 | * @param mixed $value 142 | * @return bool 143 | */ 144 | public function __set($name, $value) { 145 | // append the base options & headers 146 | if ($name == "options" || $name == "headers") { 147 | $this->{$name} = $value + $this->{$name}; 148 | } else { 149 | $this->{$name} = $value; 150 | } 151 | return true; 152 | } 153 | 154 | /** 155 | * Add a request to the request queue 156 | * 157 | * @param Request $request 158 | * @return bool 159 | */ 160 | public function add($request) { 161 | $this->requests[] = $request; 162 | return true; 163 | } 164 | 165 | /** 166 | * Create new Request and add it to the request queue 167 | * 168 | * @param string $url 169 | * @param string $method 170 | * @param $post_data 171 | * @param $headers 172 | * @param $options 173 | * @return bool 174 | */ 175 | public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) { 176 | $this->requests[] = new RollingCurlRequest($url, $method, $post_data, $headers, $options); 177 | return true; 178 | } 179 | 180 | /** 181 | * Perform GET request 182 | * 183 | * @param string $url 184 | * @param $headers 185 | * @param $options 186 | * @return bool 187 | */ 188 | public function get($url, $headers = null, $options = null) { 189 | return $this->request($url, "GET", null, $headers, $options); 190 | } 191 | 192 | /** 193 | * Perform POST request 194 | * 195 | * @param string $url 196 | * @param $post_data 197 | * @param $headers 198 | * @param $options 199 | * @return bool 200 | */ 201 | public function post($url, $post_data = null, $headers = null, $options = null) { 202 | return $this->request($url, "POST", $post_data, $headers, $options); 203 | } 204 | 205 | /** 206 | * Execute processing 207 | * 208 | * @param int $window_size Max number of simultaneous connections 209 | * @return string|bool 210 | */ 211 | public function execute($window_size = null) { 212 | // rolling curl window must always be greater than 1 213 | if (sizeof($this->requests) == 1) { 214 | return $this->single_curl(); 215 | } else { 216 | // start the rolling curl. window_size is the max number of simultaneous connections 217 | return $this->rolling_curl($window_size); 218 | } 219 | } 220 | 221 | /** 222 | * Performs a single curl request 223 | * 224 | * @access private 225 | * @return string 226 | */ 227 | private function single_curl() { 228 | $ch = curl_init(); 229 | $request = array_shift($this->requests); 230 | $options = $this->get_options($request); 231 | curl_setopt_array($ch, $options); 232 | $output = curl_exec($ch); 233 | $info = curl_getinfo($ch); 234 | 235 | // it's not neccesary to set a callback for one-off requests 236 | if ($this->callback) { 237 | $callback = $this->callback; 238 | if (is_callable($this->callback)) { 239 | call_user_func($callback, $output, $info, $request); 240 | } 241 | } 242 | else 243 | return $output; 244 | return true; 245 | } 246 | 247 | /** 248 | * Performs multiple curl requests 249 | * 250 | * @access private 251 | * @throws RollingCurlException 252 | * @param int $window_size Max number of simultaneous connections 253 | * @return bool 254 | */ 255 | private function rolling_curl($window_size = null) { 256 | if ($window_size) 257 | $this->window_size = $window_size; 258 | 259 | // make sure the rolling window isn't greater than the # of urls 260 | if (sizeof($this->requests) < $this->window_size) 261 | $this->window_size = sizeof($this->requests); 262 | 263 | if ($this->window_size < 2) { 264 | throw new RollingCurlException("Window size must be greater than 1"); 265 | } 266 | 267 | $master = curl_multi_init(); 268 | 269 | // start the first batch of requests 270 | for ($i = 0; $i < $this->window_size; $i++) { 271 | $ch = curl_init(); 272 | 273 | $options = $this->get_options($this->requests[$i]); 274 | 275 | curl_setopt_array($ch, $options); 276 | curl_multi_add_handle($master, $ch); 277 | 278 | // Add to our request Maps 279 | $key = (string) $ch; 280 | $this->requestMap[$key] = $i; 281 | } 282 | 283 | do { 284 | while (($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM) ; 285 | if ($execrun != CURLM_OK) 286 | break; 287 | // a request was just completed -- find out which one 288 | while ($done = curl_multi_info_read($master)) { 289 | 290 | // get the info and content returned on the request 291 | $info = curl_getinfo($done['handle']); 292 | $output = curl_multi_getcontent($done['handle']); 293 | 294 | // send the return values to the callback function. 295 | $callback = $this->callback; 296 | if (is_callable($callback)) { 297 | $key = (string) $done['handle']; 298 | $request = $this->requests[$this->requestMap[$key]]; 299 | unset($this->requestMap[$key]); 300 | call_user_func($callback, $output, $info, $request); 301 | } 302 | 303 | // start a new request (it's important to do this before removing the old one) 304 | if ($i < sizeof($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) { 305 | $ch = curl_init(); 306 | $options = $this->get_options($this->requests[$i]); 307 | curl_setopt_array($ch, $options); 308 | curl_multi_add_handle($master, $ch); 309 | 310 | // Add to our request Maps 311 | $key = (string) $ch; 312 | $this->requestMap[$key] = $i; 313 | $i++; 314 | } 315 | 316 | // remove the curl handle that just completed 317 | curl_multi_remove_handle($master, $done['handle']); 318 | 319 | } 320 | 321 | // Block for data in / output; error handling is done by curl_multi_exec 322 | if ($running) 323 | curl_multi_select($master, $this->timeout); 324 | 325 | } while ($running); 326 | curl_multi_close($master); 327 | return true; 328 | } 329 | 330 | 331 | /** 332 | * Helper function to set up a new request by setting the appropriate options 333 | * 334 | * @access private 335 | * @param Request $request 336 | * @return array 337 | */ 338 | private function get_options($request) { 339 | // options for this entire curl object 340 | $options = $this->__get('options'); 341 | if (ini_get('safe_mode') == 'Off' || !ini_get('safe_mode')) { 342 | $options[CURLOPT_FOLLOWLOCATION] = 1; 343 | $options[CURLOPT_MAXREDIRS] = 5; 344 | } 345 | $headers = $this->__get('headers'); 346 | 347 | // append custom options for this specific request 348 | if ($request->options) { 349 | $options = $request->options + $options; 350 | } 351 | 352 | // set the request URL 353 | $options[CURLOPT_URL] = $request->url; 354 | 355 | // posting data w/ this request? 356 | if ($request->post_data) { 357 | $options[CURLOPT_POST] = 1; 358 | $options[CURLOPT_POSTFIELDS] = $request->post_data; 359 | } 360 | if ($headers) { 361 | $options[CURLOPT_HEADER] = 0; 362 | $options[CURLOPT_HTTPHEADER] = $headers; 363 | } 364 | 365 | return $options; 366 | } 367 | 368 | /** 369 | * @return void 370 | */ 371 | public function __destruct() { 372 | unset($this->window_size, $this->callback, $this->options, $this->headers, $this->requests); 373 | } 374 | } 375 | --------------------------------------------------------------------------------