├── .gitignore ├── .travis.yml ├── BSD_LICENSE ├── CHANGELOG ├── CONTRIBUTORS ├── LICENSE ├── Makefile ├── README.md ├── bootstrap_travis.sh ├── doc ├── edoc-info ├── erlang.png ├── ibrowse.html ├── ibrowse_app.html ├── ibrowse_http_client.html ├── ibrowse_lb.html ├── ibrowse_lib.html ├── ibrowse_socks5.html ├── ibrowse_sup.html ├── index.html ├── modules-frame.html ├── overview-summary.html ├── short-desc └── stylesheet.css ├── erlang.mk ├── include └── ibrowse.hrl ├── priv └── ibrowse.conf ├── rebar.config ├── rebar.lock ├── src ├── Emakefile.src ├── ibrowse.app.src ├── ibrowse.erl ├── ibrowse_app.erl ├── ibrowse_http_client.erl ├── ibrowse_lb.erl ├── ibrowse_lib.erl ├── ibrowse_socks5.erl └── ibrowse_sup.erl └── test ├── Makefile ├── ibrowse_lib_tests.erl ├── ibrowse_load_test.erl ├── ibrowse_socks_server.erl ├── ibrowse_test.erl ├── ibrowse_test_server.erl └── ibrowse_tests.erl /.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | ebin 3 | *.beam 4 | *.sw* 5 | deps 6 | .DS_Store 7 | erl_crash.dump 8 | .eunit 9 | mime.types 10 | .rebar 11 | *.plt 12 | .rebar 13 | *~ 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | otp_release: 3 | - R16B 4 | - R16B03-1 5 | - 17.0 6 | - 17.1 7 | - 18.0 8 | - 18.1 9 | - 18.2.1 10 | - 19.3 11 | - 20.0 12 | before_script: 13 | - "./bootstrap_travis.sh" 14 | script: "./rebar3 eunit" 15 | -------------------------------------------------------------------------------- /BSD_LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2005-2014, Chandrashekhar Mullaparthi 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | * Neither the name of the T-Mobile nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 11 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | CONTRIBUTIONS & CHANGE HISTORY 2 | ============================== 3 | 4 | 21-09-2018 - v4.4.2 5 | * Fix for #163 - default to using IPv4 6 | 7 | 23-08-2018 - v4.4.1 8 | * Fixes to TLS socket handling (PR#163) 9 | * Fix ipv6 address family handling (PR#155) 10 | * Don't send messages to closed/inactive connections (PR#152) 11 | 12 | 28-01-2017 - v4.4 13 | * Fixes to SOCKS over SSL processing 14 | * Added stream_full_chunks option 15 | * Merged pull requests 145, 151 16 | 17 | 07-06-2016 - v4.3 18 | * Adopted erlang.mk for compiling. I find it easier to understand 19 | how 'make' behaves compared to rebar. This repo can still be built 20 | using rebar for those who prefer it 21 | * Removed references to lager. Introduced configurable logging function 22 | * Fixed an issue where the calling process was getting an extra 23 | spurious timeout message when the request was timing out 24 | 25 | 19-04-2016 - v4.2.4 26 | * Fixed travis-ci build as it was failing in running tests. 27 | No code changes to ibrowse 28 | 29 | 19-04-2016 - v4.2.3 30 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/143 31 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/142 32 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/139 33 | * Fixed behaviour of option preserve_status_line 34 | 35 | 25-11-2015 - v4.2.2 36 | * Fix to ibrowse.app.src to enable publishing using Hex 37 | 38 | 25-11-2015 - v4.2.1 39 | * Merged pull request https://github.com/cmullaparthi/ibrowse/pull/132 40 | * Merged pull request https://github.com/cmullaparthi/ibrowse/pull/137 41 | 42 | 28-09-2015 - v4.2 43 | * Merged long pending improvements to pipelining 44 | https://github.com/cmullaparthi/ibrowse/pull/123 45 | * Merged pull request https://github.com/cmullaparthi/ibrowse/pull/131 46 | 47 | 03-08-2015 - v4.1.2 48 | * R18 compatibility fix 49 | https://github.com/cmullaparthi/ibrowse/issues/129 50 | * Add max_attempts option 51 | https://github.com/cmullaparthi/ibrowse/pull/125 52 | * Fix for https://github.com/cmullaparthi/ibrowse/pull/120 53 | * Enhanced SOCKS5 support 54 | https://github.com/cmullaparthi/ibrowse/pull/117 55 | 56 | 10-07-2014 - v4.1.1 57 | * Added support for accepting binaries as header names 58 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/110 59 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/111 60 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/112 61 | 62 | 18-04-2013 - v4.1.0 63 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/101 64 | * Support for https://github.com/cmullaparthi/ibrowse/issues/90 65 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/86 66 | * Merged various contributions. Please see commit history for details 67 | * Introduced the return_raw_request option 68 | 69 | 09-04-2013 - v4.0.2 70 | * Tagging master with new version to cover changes 71 | contributed over the past few months via various pull requests 72 | 73 | 07-08-2012 - v4.0.1 74 | * Fix issue 67 properly. 75 | 76 | 03-08-2012 - v4.0.0 77 | * Fixed a regression in handling HEAD. 78 | https://github.com/cmullaparthi/ibrowse/issues/67 79 | 80 | * Fixed a bug in handling SSL requests through a proxy 81 | 82 | 06-04-2012 - v3.0.4 83 | * Fix for the following issue 84 | https://github.com/cmullaparthi/ibrowse/issues/67 85 | 86 | 13-03-2012 - v3.0.3 87 | * Fixes the following issues 88 | https://github.com/cmullaparthi/ibrowse/issues/64 89 | https://github.com/cmullaparthi/ibrowse/issues/63 90 | https://github.com/cmullaparthi/ibrowse/issues/62 91 | 92 | 31-01-2012 - v3.0.2 93 | * Fixed bug when stopping ibrowse. Not service affecting. 94 | 95 | 23-01-2012 - v3.0.1 96 | * Fixed bug highlighted by Dialyzer 97 | 98 | 23-01-2012 - v3.0.0 99 | * Change to the way pipelining works. 100 | * Fixed various issues reported 101 | 102 | 13-04-2011 - v2.2.0 103 | * Filipe David Manana added IPv6 support. This is a major new 104 | feature, Thank you Filipe! 105 | * Joseph Wayne Norton contributed tweaks to .gitignore 106 | 107 | 09-02-2011 - v2.1.4 108 | * Fixed a bug reported by Ryan Zezeski with the 109 | save_response_to_file option. 110 | https://github.com/cmullaparthi/ibrowse/issues#issue/33 111 | 112 | 16-01-2011 - v2.1.3 113 | * Fixed issues with streaming and chunked responses when using 114 | the 'caller controls socket' feature. See following links for 115 | details. Contributed by Filipe David Manana. 116 | https://github.com/cmullaparthi/ibrowse/pull/24 117 | https://github.com/cmullaparthi/ibrowse/pull/25 118 | https://github.com/cmullaparthi/ibrowse/pull/27 119 | https://github.com/cmullaparthi/ibrowse/pull/28 120 | https://github.com/cmullaparthi/ibrowse/pull/29 121 | 122 | * Fix for issue 32 reported by fholzhauser 123 | https://github.com/cmullaparthi/ibrowse/issues#issue/32 124 | 125 | * Fixed some dialyzer warnings. Thanks to Kostis for reporting 126 | them. 127 | 128 | 20-12-2010 - v2.1.2 129 | * Pipelining wasn't working when used in conjunction with the 130 | {stream_to, {self(), once}} option. Bug report by 131 | Filipe David Manana. 132 | 133 | 10-12-2010 - v2.1.1 134 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/issue/20 135 | by Filipe David Manana 136 | 137 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/issue/21 138 | by Filipe David Manana 139 | 140 | * Fix for https://github.com/cmullaparthi/ibrowse/issues/issue/23 141 | by Filipe David Manana 142 | 143 | * Fix for bugs when using SSL by Jo?o Lopes 144 | 145 | 25-10-2010 - v2.1.0 146 | * Fixed build on OpenSolaris. Bug report and patch from 147 | tholschuh. 148 | http://github.com/cmullaparthi/ibrowse/issues/issue/10 149 | 150 | * Fixed behaviour of inactivity_timeout option. Reported by 151 | Jo?o Lopes. 152 | http://github.com/cmullaparthi/ibrowse/issues/issue/11 153 | 154 | * Prevent atom table pollution when bogus URLs are input to 155 | ibrowse. Bug report by Jo?o Lopes. 156 | http://github.com/cmullaparthi/ibrowse/issues/issue/13 157 | 158 | * Automatically do Chunked-Transfer encoding of request body 159 | when the body is generated by a fun. Patch provided by 160 | Filipe David Manana. 161 | http://github.com/cmullaparthi/ibrowse/issues/issue/14 162 | 163 | * Depending on input options, ibrowse sometimes included multiple 164 | Content-Length headers. Bug reported by Paul J. Davis 165 | http://github.com/cmullaparthi/ibrowse/issues/issue/15 166 | 167 | * Deal with webservers which do not provide a Reason-Phrase on the 168 | response Status-Line. Patch provided by Jeroen Koops. 169 | http://github.com/cmullaparthi/ibrowse/issues/issue/16 170 | 171 | * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/17 172 | This was reported by Filipe David Manana. 173 | 174 | * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/19 175 | This was reported by Dan Kelley and Filipe David Manana. 176 | 177 | * Added ibrowse:stream_close/1 to close the connection 178 | associated with a certain response stream. Patch provided by 179 | Jo?o Lopes. 180 | 181 | * Prevent port number being included in the Host header when port 182 | 443 is intended. Bug reported by Andrew Tunnell-Jones 183 | 184 | 24-09-2010 - v2.0.1 185 | * Removed a spurious io:format statement 186 | 187 | 22-09-2010 - v2.0.0. 188 | 189 | * Added option preserve_chunked_encoding. This allows the 190 | caller to get the raw HTTP response when the 191 | Transfer-Encoding is Chunked. This feature was requested 192 | by Benoit Chesneau who wanted to write a HTTP proxy using 193 | ibrowse. 194 | 195 | * Fixed bug with the {stream_to, {Pid, once}} option. Bug 196 | report and lot of help from Filipe David Manana. Thank 197 | you Filipe. 198 | 199 | * The {error, conn_failed} and {error, send_failed} return 200 | values are now of the form {error, {conn_failed, Err}} 201 | and {error, {send_failed, Err}}. This is so that the 202 | specific socket error can be returned to the caller. I 203 | think it looks a bit ugly, but that is the best 204 | compromise I could come up with. 205 | 206 | * Added application configuration parameters 207 | default_max_sessions and default_max_pipeline_size. These 208 | were previously hard coded to 10. 209 | 210 | * Versioning of ibrowse now follows the Semantic Versioning 211 | principles. See http://semver.org. Thanks to Anthony 212 | Molinaro for nudging me in this direction. 213 | 214 | * The connect_timeout option now only applies to the 215 | connection setup phase. In previous versions, the time 216 | taken to setup the connection was deducted from the 217 | specified timeout value for the request. 218 | 219 | 17-07-2010 - * Merged change made by Filipe David Manana to use the base64 220 | module for encoding/decoding. 221 | 222 | 11-06-2010 - * Removed use of deprecated concat_binary. Patch supplied by 223 | Steve Vinoski 224 | 225 | 10-06-2010 - * Fixed bug in https requests not going via the proxy 226 | 227 | 12-05-2010 - * Added support for the CONNECT method to tunnel HTTPS through 228 | a proxy. When a https URL is requested through a proxy, 229 | ibrowse will automatically use the CONNECT method to first 230 | setup a tunnel through the proxy. Once this succeeds, the 231 | actual request is dispatched. Successfully tested with the 232 | new SSL implementation in R13B-03 233 | * Added SSL support for direct connections. 234 | See ibrowse:spawn_worker_process/1 and 235 | ibrowse:spawn_link_worker_process/1 236 | * Added option to return raw status line and raw unparsed headers 237 | 238 | 23-04-2010 - * Fixes to URL parsing by Karol Skocik 239 | 240 | 08-11-2009 - * Added option headers_as_is 241 | 242 | 04-10-2009 - * Patch from Kostis Sagonas to cleanup some code and suppress 243 | dialyzer warnings 244 | 245 | 24-09-2009 - * When a filename was supplied with the 'save_response_to_file' 246 | option, the option was being ignored. Bug report from 247 | Adam Kocoloski 248 | 249 | 05-09-2009 - * Introduced option to allow caller to set socket options. 250 | 251 | 29-07-2009 - * The ETS table created for load balancing of requests was not 252 | being deleted which led to the node not being able to create 253 | any more ETS tables if queries were made to many number of 254 | webservers. ibrowse now deletes the ETS table it creates once the 255 | last connection to a webserver is dropped. 256 | Reported by Seth Falcon. 257 | * Spurious data being returned at end of body in certain cases of 258 | chunked encoded responses from the server. 259 | Reported by Chris Newcombe. 260 | 261 | 03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller 262 | to control when it wants to receive more data. If this option 263 | is used, the call ibrowse:stream_next(Req_id) should be used 264 | to get more data. 265 | * Patch submitted by Steve Vinoski to remove compiler warnings 266 | about the use of obsolete guards 267 | 268 | 29-06-2009 - * Fixed following issues reported by Oscar Hellstr?m 269 | * Use {active, once} instead of {active, true} 270 | * Fix 'dodgy' timeout handling 271 | * Use binaries internally instead of lists to reduce memory 272 | consumption on 64 bit platforms. The default response format 273 | is still 'list' to maintain backwards compatibility. Use the 274 | option {response_format, binary} to get responses as binaries. 275 | * Fixed chunking bug (reported by Adam Kocoloski) 276 | * Added new option {inactivity_timeout, Milliseconds} to timeout 277 | requests if no data is received on the link for the specified 278 | interval. Useful when responses are large and links are flaky. 279 | * Added ibrowse:all_trace_off/0 to turn off all tracing 280 | * Change to the way responses to asynchronous requests are 281 | returned. The following messages have been removed. 282 | * {ibrowse_async_response, Req_id, {chunk_start, Chunk_size}} 283 | * {ibrowse_async_response, Req_id, chunk_end} 284 | * Fixed Makefiles as part of Debian packaging 285 | (thanks to Thomas Lindgren) 286 | * Moved repository from Sourceforge to Github 287 | 288 | 11-06-2009 - * Added option to control size of streamed chunks. Also added 289 | option for the client to receive responses in binary format. 290 | 291 | 21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file. 292 | Reported by Erik Reitsma on the erlyaws mailing list 293 | * Fixed bug when cleaning up closing connections 294 | 295 | 27-03-2008 - * Major rewrite of the load balancing feature. Additional module, 296 | ibrowse_lb.erl, introduced to achieve this. 297 | * Can now get a handle to a connection process which is not part of 298 | the load balancing pool. Useful when an application is making 299 | requests to a webserver which are time consuming (such as 300 | uploading a large file). Such requests can be put on a separate 301 | connection, and all other smaller/quicker requests can use the 302 | load balancing pool. See ibrowse:spawn_worker_process/2 and 303 | ibrowse:spawn_link_worker_process/2 304 | * Ram Krishnan sent a patch to enable a client to send a lot of 305 | data in a request by providing a fun which is invoked by the 306 | connection handling process. This fun can fetch the data from 307 | any where. This is useful when trying to upload a large file 308 | to a webserver. 309 | * Use the TCP_NODELAY option on every socket by default 310 | * Rudimentary support for load testing of ibrowse. Undocumented, 311 | but see ibrowse_test:load_test/3. Use the source, Luke! 312 | * New function ibrowse:show_dest_status/2 to view state of 313 | connections/pipelines to a web server 314 | 315 | 20-02-2008 - Ram Krishnan sent another patch for another hidden bug in the 316 | save_response_to_file feature. 317 | 318 | 07-02-2008 - Ram Krishnan (kriyative _at_ gmail dot com) sent a simple patch to 319 | enable specifying the filename in the save_response_to_file option. 320 | When testing the patch, I realised that my original implementation 321 | of this feature was quite flaky and a lot of corner cases were 322 | not covered. Fixed all of them. Thanks Ram! 323 | 324 | 17-10-2007 - Matthew Reilly (matthew dot reilly _at_ sipphone dot com) 325 | sent a bug report and a fix. If the chunk trailer spans two TCP 326 | packets, then ibrowse fails to recognise that the chunked transfer 327 | has ended. 328 | 329 | 29-08-2007 - Bug report by Peter Kristensen(ptx _at_ daimi dot au dot dk). 330 | ibrowse crashes when the webserver returns just the Status line 331 | and nothing else. 332 | 333 | 28-06-2007 - Added host_header option to enable connection to secure sites 334 | via stunnel 335 | 336 | 20-04-2007 - Geoff Cant sent a patch to remove URL encoding for digits in 337 | ibrowse_lib:url_encode/1. 338 | ibrowse had a dependency on the inets application because the 339 | ibrowse_http_client.erl invoked httpd_util:encode_base64/1. This 340 | dependency is now removed and the encode_base64/1 has been 341 | implemented in ibrowse_lib.erl 342 | 343 | 06-03-2007 - Eric Merritt sent a patch to support WebDAV requests. 344 | 345 | 12-01-2007 - Derek Upham sent in a bug fix. The reset_state function was not 346 | behaving correctly when the transfer encoding was not chunked. 347 | 348 | 13-11-2006 - Youn?s Hafri reported a bug where ibrowse was not returning the 349 | temporary filename when the server was closing the connection 350 | after sending the data (as in HTTP/1.0). 351 | Released ibrowse under the BSD license 352 | 353 | 12-10-2006 - Chris Newcombe reported bug in dealing with requests where no 354 | body is expected in the response. The first request would succeed 355 | and the next request would hang. 356 | 357 | 24-May-2006 - Sean Hinde reported a bug. Async responses with pipelining was 358 | returning the wrong result. 359 | 360 | 08-Dec-2005 - Richard Cameron (camster@citeulike.org). Patch to ibrowse to 361 | prevent port number being included in the Host header when port 362 | 80 is intended. 363 | 364 | 22-Nov-2005 - Added ability to generate requests using the Chunked 365 | Transfer-Encoding. 366 | 367 | 08-May-2005 - Youn?s Hafri made a CRUX LINUX port of ibrowse. 368 | http://yhafri.club.fr/crux/index.html 369 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | CONTRIBUTORS 2 | ============ 3 | The following people have helped maked ibrowse better by reporting bugs, 4 | supplying patches and also asking for new features. Please write to me if you 5 | have contributed and I've missed you out. 6 | 7 | In alphabetical order: 8 | 9 | Adam Kocoloski 10 | Andrew Tunnell-Jones 11 | Anthony Molinaro 12 | Benjamin P Lee (https://github.com/benjaminplee) 13 | Benoit Chesneau (https://github.com/benoitc) 14 | Brian Richards (http://github.com/richbria) 15 | Chris Newcombe 16 | Dan Kelley 17 | Dan Schwabe (https://github.com/dfschwabe) 18 | Dave Cottlehuber 19 | Derek Upham 20 | Eric Merritt 21 | Erik Reitsma 22 | Fabian MuscarielloG 23 | Filipe David Manana 24 | Geoff Cant 25 | Jeroen Koops 26 | Jo?o Lopes 27 | Joseph Wayne Norton 28 | Karol Skocik 29 | Konstantin Nikiforov 30 | Kostis Sagonas 31 | Marcelo Gornstein (https://github.com/marcelog) 32 | Matthew Reilly 33 | Michael Terry 34 | Oscar Hellstr?m 35 | Paul J. Davis 36 | Peter Kristensen 37 | Ram Krishnan 38 | Richard Cameron 39 | Robert Newson (https://github.com/rnewson) 40 | Ryan Flynn 41 | Ryan Zezeski 42 | Sean Hinde 43 | Serge Polkovnikov (https://github.com/serge2) 44 | Sergey Samokhi 45 | Seth Falcon 46 | Steve Vinoski 47 | Thomas Lindgren 48 | Vincent Ambo 49 | Youn?s Hafri 50 | Yury Gargay (https://github.com/surik) 51 | fholzhauser (https://github.com/fholzhauser/) 52 | getong (https://github.com/getong) 53 | lissana (https://github.com/lissana) 54 | hyperthunk (https://github.com/hyperthunk/) 55 | Mistagrooves (https://github.com/Mistagrooves/) 56 | tholschuh (https://github.com/tholschuh/) 57 | https://github.com/apauley 58 | https://github.com/AeroNotix 59 | https://github.com/dis 60 | https://github.com/f355 61 | https://github.com/flycodepl 62 | https://github.com/helllamer 63 | https://github.com/marutha 64 | https://github.com/nrdufour 65 | https://github.com/pib 66 | https://github.com/puzza007 67 | https://github.com/rflynn 68 | https://github.com/Vagabond 69 | https://github.com/divolgin 70 | https://github.com/vans163 71 | https://github.com/shakugan 72 | https://github.com/p2k 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ibrowse - a HTTP client written in erlang 2 | Copyright (C) 2005-2014 Chandrashekhar Mullaparthi 3 | 4 | This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. 5 | 6 | This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. 7 | 8 | You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 9 | 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT=ibrowse 2 | PLT_APPS=erts kernel stdlib ssl crypto public_key 3 | TEST_ERLC_OPTS=-pa ../ibrowse/ebin 4 | 5 | include erlang.mk 6 | 7 | test: app eunit unit_tests old_tests 8 | @echo "=====================================================" 9 | 10 | unit_tests: 11 | @echo "=====================================================" 12 | @echo "Running tests..." 13 | @cd test && make test && cd .. 14 | 15 | old_tests: 16 | @echo "=====================================================" 17 | @echo "Running old tests..." 18 | @cd test && make old_tests && cd .. 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ibrowse [![Build Status](https://secure.travis-ci.org/cmullaparthi/ibrowse.svg?branch=master)](http://travis-ci.org/cmullaparthi/ibrowse) 2 | 3 | ibrowse is a HTTP client written in erlang. 4 | 5 | **License:** ibrowse is available under two different licenses. 6 | LGPL or the BSD license. 7 | 8 | **Comments to:** chandrashekhar.mullaparthi@gmail.com 9 | 10 | **Current Version:** 4.4.2 11 | 12 | **Latest Version:** git://github.com/cmullaparthi/ibrowse.git 13 | 14 | 15 | 16 | ## Features 17 | 18 | * [RFC2616](http://www.ietf.org/rfc/rfc2616.txt) compliant (AFAIK) 19 | * supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE, 20 | MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK, MOVE and COPY 21 | * Understands HTTP/0.9, HTTP/1.0 and HTTP/1.1 22 | * Understands chunked encoding 23 | * Can generate requests using [Chunked Transfer-Encoding](http://en.wikipedia.org/wiki/Chunked_transfer_encoding) 24 | * Pools of connections to each webserver 25 | * Pipelining support 26 | * Download to file 27 | * Asynchronous requests. Responses are streamed to a process 28 | * Basic authentication 29 | * Supports proxy authentication 30 | * Supports SOCKS5 31 | * Authentication methods 0 (No authentication) and 2(Username/password) supported 32 | * Can talk to secure webservers using SSL 33 | * *Any other features in the code not listed here :)* 34 | 35 | 36 | 37 | ## Usage Examples 38 | 39 | Remember to start ibrowse first: 40 | 41 | ```erlang 42 | 5> ibrowse:start(). 43 | {ok,<0.94.0>} 44 | ``` 45 | 46 | 47 | 48 | ### Synchronous Requests 49 | 50 | A simple `GET` request: 51 | 52 | ```erlang 53 | 6> ibrowse:send_req("http://intranet/messenger/", [], get). 54 | {ok,"200", 55 | [{"Server","Microsoft-IIS/5.0"}, 56 | {"Content-Location","http://intranet/messenger/index.html"}, 57 | {"Date","Fri, 17 Dec 2004 15:16:19 GMT"}, 58 | {"Content-Type","text/html"}, 59 | {"Accept-Ranges","bytes"}, 60 | {"Last-Modified","Fri, 17 Dec 2004 08:38:21 GMT"}, 61 | {"Etag","\"aa7c9dc313e4c41:d77\""}, 62 | {"Content-Length","953"}], 63 | "..."} 64 | ``` 65 | 66 | 67 | A `GET` using a proxy: 68 | 69 | ```erlang 70 | 7> ibrowse:send_req("http://www.google.com/", [], get, [], 71 | [{proxy_user, "XXXXX"}, 72 | {proxy_password, "XXXXX"}, 73 | {proxy_host, "proxy"}, 74 | {proxy_port, 8080}], 1000). 75 | {ok,"302", 76 | [{"Date","Fri, 17 Dec 2004 15:22:56 GMT"}, 77 | {"Content-Length","217"}, 78 | {"Content-Type","text/html"}, 79 | {"Set-Cookie", 80 | "PREF=ID=f58155c797f9..."}, 81 | {"Server","GWS/2.1"}, 82 | {"Location", 83 | "http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D110329..."}, 84 | {"Via","1.1 netapp01 (NetCache NetApp/5.5R2)"}], 85 | "...\r\n"} 86 | ``` 87 | 88 | 89 | A `GET` response saved to file. A temporary file is created and the 90 | filename returned. The response will only be saved to file if the 91 | status code is in the `200` range. The directory to download to can 92 | be set using the application env var `download_dir` - the default 93 | is the current working directory: 94 | 95 | ```erlang 96 | 8> ibrowse:send_req("http://www.erlang.se/", [], get, [], 97 | [{proxy_user, "XXXXX"}, 98 | {proxy_password, "XXXXX"}, 99 | {proxy_host, "proxy"}, 100 | {proxy_port, 8080}, 101 | {save_response_to_file, true}], 1000). 102 | {error,req_timedout} 103 | 104 | 9> ibrowse:send_req("http://www.erlang.se/", [], get, [], 105 | [{proxy_user, "XXXXX"}, 106 | {proxy_password, "XXXXX"}, 107 | {proxy_host, "proxy"}, 108 | {proxy_port, 8080}, 109 | {save_response_to_file, true}], 5000). 110 | {ok,"200", 111 | [{"Transfer-Encoding","chunked"}, 112 | {"Date","Fri, 17 Dec 2004 15:24:36 GMT"}, 113 | {"Content-Type","text/html"}, 114 | {"Server","Apache/1.3.9 (Unix)"}, 115 | {"Via","1.1 netapp01 (NetCache NetApp/5.5R2)"}], 116 | {file,"/Users/chandru/code/ibrowse/src/ibrowse_tmp_file_1103297041125854"}} 117 | ``` 118 | 119 | 120 | Setting the size of the connection pool and pipeline. This sets the 121 | number of maximum connections to the specified server to `10` and the pipeline 122 | size to `1`. Connections are assumed to be already setup. 123 | 124 | ```erlang 125 | 11> ibrowse:set_dest("www.hotmail.com", 80, [{max_sessions, 10}, 126 | {max_pipeline_size, 1}]). 127 | ok 128 | ``` 129 | 130 | 131 | Example using the `HEAD` method: 132 | 133 | ```erlang 134 | 56> ibrowse:send_req("http://www.erlang.org", [], head). 135 | {ok,"200", 136 | [{"Date","Mon, 28 Feb 2005 04:40:53 GMT"}, 137 | {"Server","Apache/1.3.9 (Unix)"}, 138 | {"Last-Modified","Thu, 10 Feb 2005 09:31:23 GMT"}, 139 | {"Etag","\"8d71d-1efa-420b29eb\""}, 140 | {"Accept-ranges","bytes"}, 141 | {"Content-Length","7930"}, 142 | {"Content-Type","text/html"}], 143 | []} 144 | ``` 145 | 146 | 147 | Example using the `OPTIONS` method: 148 | 149 | ```erlang 150 | 62> ibrowse:send_req("http://www.sun.com", [], options). 151 | {ok,"200", 152 | [{"Server","Sun Java System Web Server 6.1"}, 153 | {"Date","Mon, 28 Feb 2005 04:44:39 GMT"}, 154 | {"Content-Length","0"}, 155 | {"P3p", 156 | "policyref=\"http://www.sun.com/p3p/Sun_P3P_Policy.xml\", CP=\"CAO DSP COR CUR ADMa DEVa TAIa PSAa PSDa CONi TELi OUR SAMi PUBi IND PHY ONL PUR COM NAV INT DEM CNT STA POL PRE GOV\""}, 157 | {"Set-Cookie", 158 | "SUN_ID=X.X.X.X:169191109565879; EXPIRES=Wednesday, 31-Dec-2025 23:59:59 GMT; DOMAIN=.sun.com; PATH=/"}, 159 | {"Allow", 160 | "HEAD, GET, PUT, POST, DELETE, TRACE, OPTIONS, MOVE, INDEX, MKDIR, RMDIR"}], 161 | []} 162 | ``` 163 | 164 | 165 | 166 | ### Asynchronous Requests 167 | 168 | Example of an asynchronous `GET` request: 169 | 170 | ```erlang 171 | 18> ibrowse:send_req("http://www.google.com", [], get, [], 172 | [{proxy_user, "XXXXX"}, 173 | {proxy_password, "XXXXX"}, 174 | {proxy_host, "proxy"}, 175 | {proxy_port, 8080}, 176 | {stream_to, self()}]). 177 | {ibrowse_req_id,{1115,327256,389608}} 178 | 179 | 19> flush(). 180 | Shell got {ibrowse_async_headers,{1115,327256,389608}, 181 | "302", 182 | [{"Date","Thu, 05 May 2005 21:06:41 GMT"}, 183 | {"Content-Length","217"}, 184 | {"Content-Type","text/html"}, 185 | {"Set-Cookie", 186 | "PREF=ID=b601f16bfa32f071:CR=1:TM=1115327201:LM=1115327201:S=OX5hSB525AMjUUu7; expires=Sun, 17-Jan-2038 19:14:07 GMT; path=/; domain=.google.com"}, 187 | {"Server","GWS/2.1"}, 188 | {"Location", 189 | "http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D1115327201:S%3DDS9pDJ4IHcAuZ_AS&prev=/"}, 190 | {"Via", 191 | "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]} 192 | Shell got {ibrowse_async_response,{1115,327256,389608}, 193 | "...\r\n"} 194 | Shell got {ibrowse_async_response_end,{1115,327256,389608}} 195 | ok 196 | ``` 197 | 198 | 199 | Another asynchronous `GET` request: 200 | 201 | ```erlang 202 | 24> ibrowse:send_req("http://yaws.hyber.org/simple_ex2.yaws", [], get, [], 203 | [{proxy_user, "XXXXX"}, 204 | {proxy_password, "XXXXX"}, 205 | {proxy_host, "proxy"}, 206 | {proxy_port, 8080}, 207 | {stream_to, self()}]). 208 | {ibrowse_req_id,{1115,327430,512314}} 209 | 210 | 25> flush(). 211 | Shell got {ibrowse_async_headers,{1115,327430,512314}, 212 | "200", 213 | [{"Date","Thu, 05 May 2005 20:58:08 GMT"}, 214 | {"Content-Length","64"}, 215 | {"Content-Type","text/html;charset="}, 216 | {"Server", 217 | "Yaws/1.54 Yet Another Web Server"}, 218 | {"Via", 219 | "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]} 220 | Shell got {ibrowse_async_response,{1115,327430,512314}, 221 | "...\n"} 222 | Shell got {ibrowse_async_response_end,{1115,327430,512314}} 223 | ``` 224 | 225 | 226 | Example of request which fails when using the async option. Here 227 | the `{ibrowse_req_id, ReqId}` is not returned. Instead the error code is 228 | returned. 229 | 230 | ```erlang 231 | 68> ibrowse:send_req("http://www.earlyriser.org", [], get, [], [{stream_to, self()}]). 232 | {error,conn_failed} 233 | ``` 234 | 235 | 236 | 237 | ### Other Examples 238 | 239 | Example of request using both Proxy-Authorization and authorization 240 | by the final webserver: 241 | 242 | ```erlang 243 | 17> ibrowse:send_req("http://www.erlang.se/lic_area/protected/patches/erl_756_otp_beam.README", 244 | [], get, [], 245 | [{proxy_user, "XXXXX"}, 246 | {proxy_password, "XXXXX"}, 247 | {proxy_host, "proxy"}, 248 | {proxy_port, 8080}, 249 | {basic_auth, {"XXXXX", "XXXXXX"}}]). 250 | {ok,"200", 251 | [{"Accept-Ranges","bytes"}, 252 | {"Date","Thu, 05 May 2005 21:02:09 GMT"}, 253 | {"Content-Length","2088"}, 254 | {"Content-Type","text/plain"}, 255 | {"Server","Apache/1.3.9 (Unix)"}, 256 | {"Last-Modified","Tue, 03 May 2005 15:08:18 GMT"}, 257 | {"ETag","\"1384c8-828-427793e2\""}, 258 | {"Via","1.1 hatproxy01 (NetCache NetApp/5.6.2)"}], 259 | "Patch Id:\t\terl_756_otp_beam\n..."} 260 | ``` 261 | 262 | 263 | Example of a `TRACE` request. Very interesting! yaws.hyber.org didn't 264 | support this. Nor did www.google.com. But good old BBC supports this: 265 | 266 | ```erlang 267 | 37> ibrowse:send_req("http://www.bbc.co.uk/", [], trace, [], 268 | [{proxy_user, "XXXXX"}, 269 | {proxy_password, "XXXXX"}, 270 | {proxy_host, "proxy"}, 271 | {proxy_port, 8080}]). 272 | {ok,"200", 273 | [{"Transfer-Encoding","chunked"}, 274 | {"Date","Thu, 05 May 2005 21:40:27 GMT"}, 275 | {"Content-Type","message/http"}, 276 | {"Server","Apache/2.0.51 (Unix)"}, 277 | {"Set-Cookie", 278 | "BBC-UID=7452e72a..."}, 279 | {"Set-Cookie", 280 | "BBC-UID=7452e72a..."}, 281 | {"Via","1.1 hatproxy01 (NetCache NetApp/5.6.2)"}], 282 | "TRACE / HTTP/1.1\r\nHost: www.bbc.co.uk\r\nConnection: keep-alive\r\nX-Forwarded-For: 172.24.28.29\r\nVia: 1.1 hatproxy01 (NetCache NetApp/5.6.2)\r\nCookie: BBC-UID=7452e...\r\n\r\n"} 283 | ``` 284 | 285 | A `GET` using a socks5: 286 | 287 | ```erlang 288 | ibrowse:send_req("http://google.com", [], get, [], 289 | [{socks5_host, "127.0.0.1"}, 290 | {socks5_port, 5335}]). 291 | 292 | ibrowse:send_req("http://google.com", [], get, [], 293 | [{socks5_host, "127.0.0.1"}, 294 | {socks5_port, 5335}, 295 | {socks5_user, "user4321"}, 296 | {socks5_password, "pass7654"}]). 297 | ``` 298 | -------------------------------------------------------------------------------- /bootstrap_travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -O -L https://s3.amazonaws.com/rebar3/rebar3 4 | chmod +x rebar3 5 | ./rebar3 update -------------------------------------------------------------------------------- /doc/edoc-info: -------------------------------------------------------------------------------- 1 | %% encoding: UTF-8 2 | {application,ibrowse}. 3 | {modules,[ibrowse,ibrowse_app,ibrowse_http_client,ibrowse_lb,ibrowse_lib, 4 | ibrowse_socks5,ibrowse_sup]}. 5 | -------------------------------------------------------------------------------- /doc/erlang.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cmullaparthi/ibrowse/22d6fd6baa6e83633aa2923f41589945c1d2dc2f/doc/erlang.png -------------------------------------------------------------------------------- /doc/ibrowse.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse

13 | The ibrowse application implements an HTTP 1.1 client in erlang. 14 |

Copyright © 2005-2014 Chandrashekhar Mullaparthi

15 | 16 |

Behaviours: gen_server.

17 |

Authors: Chandrashekhar Mullaparthi (chandrashekhar dot mullaparthi at gmail dot com).

18 | 19 |

Description

The ibrowse application implements an HTTP 1.1 client in erlang. This 20 | module implements the API of the HTTP client. There is one named 21 | process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is 22 | one process to handle one TCP connection to a webserver 23 | (implemented in the module ibrowse_http_client). Multiple connections to a 24 | webserver are setup based on the settings for each webserver. The 25 | ibrowse process also determines which connection to pipeline a 26 | certain request on. The functions to call are send_req/3, 27 | send_req/4, send_req/5, send_req/6.

28 | 29 |

Here are a few sample invocations.

30 | 31 | 32 | ibrowse:send_req("http://intranet/messenger/", [], get). 33 |

34 | 35 | ibrowse:send_req("http://www.google.com/", [], get, [], 36 | [{proxy_user, "XXXXX"}, 37 | {proxy_password, "XXXXX"}, 38 | {proxy_host, "proxy"}, 39 | {proxy_port, 8080}], 1000). 40 |

41 | 42 | ibrowse:send_req("http://www.erlang.org/download/otp_src_R10B-3.tar.gz", [], get, [], 43 | [{proxy_user, "XXXXX"}, 44 | {proxy_password, "XXXXX"}, 45 | {proxy_host, "proxy"}, 46 | {proxy_port, 8080}, 47 | {save_response_to_file, true}], 1000). 48 |

49 | 50 | ibrowse:send_req("http://www.erlang.org", [], head). 51 | 52 |

53 | ibrowse:send_req("http://www.sun.com", [], options). 54 | 55 |

56 | ibrowse:send_req("http://www.bbc.co.uk", [], trace). 57 | 58 |

59 | ibrowse:send_req("http://www.google.com", [], get, [], 60 | [{stream_to, self()}]). 61 | 62 | 63 |

Function Index

64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 77 | 78 | 79 | 80 | 81 | 82 | 84 | 86 | 88 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 98 | 100 | 101 | 103 | 104 | 105 | 106 | 107 | 109 | 111 | 113 | 114 | 115 | 117 | 118 | 120 |
add_config/1Add additional configuration elements at runtime.
all_trace_off/0Turn Off ALL tracing.
code_change/3
get_config_value/1Internal export.
get_config_value/2Internal export.
get_metrics/0
get_metrics/2
handle_call/3
handle_cast/2
handle_info/2
init/1
rescan_config/0Clear current configuration for ibrowse and load from the file 76 | ibrowse.conf in the IBROWSE_EBIN/../priv directory.
rescan_config/1
send_req/3This is the basic function to send a HTTP request.
send_req/4Same as send_req/3.
send_req/5Same as send_req/4.
send_req/6Same as send_req/5.
send_req_direct/4Same as send_req/3 except that the first argument is the PID 83 | returned by spawn_worker_process/2 or spawn_link_worker_process/2.
send_req_direct/5Same as send_req/4 except that the first argument is the PID 85 | returned by spawn_worker_process/2 or spawn_link_worker_process/2.
send_req_direct/6Same as send_req/5 except that the first argument is the PID 87 | returned by spawn_worker_process/2 or spawn_link_worker_process/2.
send_req_direct/7Same as send_req/6 except that the first argument is the PID 89 | returned by spawn_worker_process/2 or spawn_link_worker_process/2.
set_dest/3Deprecated.
set_max_attempts/3Set the maximum attempts for each connection to a specific Host:Port.
set_max_pipeline_size/3Set the maximum pipeline size for each connection to a specific Host:Port.
set_max_sessions/3Set the maximum number of connections allowed to a specific Host:Port.
show_dest_status/0Shows some internal information about load balancing.
show_dest_status/1
show_dest_status/2Shows some internal information about load balancing to a 97 | specified Host:Port.
spawn_link_worker_process/1Same as spawn_worker_process/1 except the the calling process 99 | is linked to the worker process which is spawned.
spawn_link_worker_process/2Same as spawn_link_worker_process/1 except with Erlang process options.
spawn_worker_process/1Creates a HTTP client process to the specified Host:Port which 102 | is not part of the load balancing pool.
spawn_worker_process/2Same as spawn_worker_process/1 except with Erlang process options.
start/0Starts the ibrowse process without linking.
start_link/0Starts the ibrowse process linked to the calling process.
stop/0Stop the ibrowse process.
stop_worker_process/1Terminate a worker process spawned using 108 | spawn_worker_process/2 or spawn_link_worker_process/2.
stream_close/1Tell ibrowse to close the connection associated with the 110 | specified stream.
stream_next/1Tell ibrowse to stream the next chunk of data to the 112 | caller.
terminate/2
trace_off/0Turn tracing off for the ibrowse process.
trace_off/2Turn tracing OFF for all connections to the specified HTTP 116 | server.
trace_on/0Turn tracing on for the ibrowse process.
trace_on/2Turn tracing on for all connections to the specified HTTP 119 | server.
121 | 122 |

Function Details

123 | 124 |

add_config/1

125 |
126 |

add_config(Terms) -> any()

127 |

Add additional configuration elements at runtime.

128 | 129 |

all_trace_off/0

130 |
131 |

all_trace_off() -> ok

132 |

Turn Off ALL tracing

133 | 134 |

code_change/3

135 |
136 |

code_change(OldVsn, State, Extra) -> any()

137 |
138 | 139 |

get_config_value/1

140 |
141 |

get_config_value(Key) -> any()

142 |

Internal export

143 | 144 |

get_config_value/2

145 |
146 |

get_config_value(Key, DefVal) -> any()

147 |

Internal export

148 | 149 |

get_metrics/0

150 |
151 |

get_metrics() -> any()

152 |
153 | 154 |

get_metrics/2

155 |
156 |

get_metrics(Host, Port) -> any()

157 |
158 | 159 |

handle_call/3

160 |
161 |

handle_call(Request, From, State) -> any()

162 |
163 | 164 |

handle_cast/2

165 |
166 |

handle_cast(Msg, State) -> any()

167 |
168 | 169 |

handle_info/2

170 |
171 |

handle_info(Info, State) -> any()

172 |
173 | 174 |

init/1

175 |
176 |

init(X1) -> any()

177 |
178 | 179 |

rescan_config/0

180 |
181 |

rescan_config() -> any()

182 |

Clear current configuration for ibrowse and load from the file 183 | ibrowse.conf in the IBROWSE_EBIN/../priv directory. Current 184 | configuration is cleared only if the ibrowse.conf file is readable 185 | using file:consult/1

186 | 187 |

rescan_config/1

188 |
189 |

rescan_config(Terms) -> any()

190 |
191 | 192 |

send_req/3

193 |
194 |

send_req(Url::string(), Headers::headerList(), Method::method()) -> response() 195 |

196 |

This is the basic function to send a HTTP request. 197 | The Status return value indicates the HTTP status code returned by the webserver

198 | 199 |

send_req/4

200 |
201 |

send_req(Url, Headers, Method::method(), Body::body()) -> response() 202 |

203 |

Same as send_req/3. 204 | If a list is specified for the body it has to be a flat list. The body can also be a fun/0 or a fun/1.
205 | If fun/0, the connection handling process will repeatdely call the fun until it returns an error or eof. 

Fun() = {ok, Data} | eof

206 | If fun/1, the connection handling process will repeatedly call the fun with the supplied state until it returns an error or eof. 
Fun(State) = {ok, Data} | {ok, Data, NewState} | eof

207 | 208 |

send_req/5

209 |
210 |

send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response() 211 |

212 |

Same as send_req/4.

213 | 214 |

send_req/6

215 |
216 |

send_req(Url, Headers::headerList(), Method::method(), Body::body(), Options::optionList(), Timeout) -> response() 217 |

218 |

Same as send_req/5. 219 | All timeout values are in milliseconds.

220 | 221 |

send_req_direct/4

222 |
223 |

send_req_direct(Conn_pid, Url, Headers, Method) -> any()

224 |

Same as send_req/3 except that the first argument is the PID 225 | returned by spawn_worker_process/2 or spawn_link_worker_process/2

226 | 227 |

send_req_direct/5

228 |
229 |

send_req_direct(Conn_pid, Url, Headers, Method, Body) -> any()

230 |

Same as send_req/4 except that the first argument is the PID 231 | returned by spawn_worker_process/2 or spawn_link_worker_process/2

232 | 233 |

send_req_direct/6

234 |
235 |

send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) -> any()

236 |

Same as send_req/5 except that the first argument is the PID 237 | returned by spawn_worker_process/2 or spawn_link_worker_process/2

238 | 239 |

send_req_direct/7

240 |
241 |

send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) -> any()

242 |

Same as send_req/6 except that the first argument is the PID 243 | returned by spawn_worker_process/2 or spawn_link_worker_process/2

244 | 245 |

set_dest/3

246 |
247 |

set_dest(Host, Port, T) -> any()

248 |

Deprecated. Use set_max_sessions/3 and set_max_pipeline_size/3 249 | for achieving the same effect.

250 | 251 |

set_max_attempts/3

252 |
253 |

set_max_attempts(Host::string(), Port::integer(), Max::integer()) -> ok

254 |

Set the maximum attempts for each connection to a specific Host:Port.

255 | 256 |

set_max_pipeline_size/3

257 |
258 |

set_max_pipeline_size(Host::string(), Port::integer(), Max::integer()) -> ok

259 |

Set the maximum pipeline size for each connection to a specific Host:Port.

260 | 261 |

set_max_sessions/3

262 |
263 |

set_max_sessions(Host::string(), Port::integer(), Max::integer()) -> ok

264 |

Set the maximum number of connections allowed to a specific Host:Port.

265 | 266 |

show_dest_status/0

267 |
268 |

show_dest_status() -> any()

269 |

Shows some internal information about load balancing. Info 270 | about workers spawned using spawn_worker_process/2 or 271 | spawn_link_worker_process/2 is not included.

272 | 273 |

show_dest_status/1

274 |
275 |

show_dest_status(Url) -> any()

276 |
277 | 278 |

show_dest_status/2

279 |
280 |

show_dest_status(Host, Port) -> any()

281 |

Shows some internal information about load balancing to a 282 | specified Host:Port. Info about workers spawned using 283 | spawn_worker_process/2 or spawn_link_worker_process/2 is not 284 | included.

285 | 286 |

spawn_link_worker_process/1

287 |
288 |

spawn_link_worker_process(Url::string() | {Host::string(), Port::integer()}) -> {ok, pid()}

289 |

Same as spawn_worker_process/1 except the the calling process 290 | is linked to the worker process which is spawned.

291 | 292 |

spawn_link_worker_process/2

293 |
294 |

spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()}

295 |

Same as spawn_link_worker_process/1 except with Erlang process options.

296 | 297 |

spawn_worker_process/1

298 |
299 |

spawn_worker_process(Url::string() | {Host::string(), Port::integer()}) -> {ok, pid()}

300 |

Creates a HTTP client process to the specified Host:Port which 301 | is not part of the load balancing pool. This is useful in cases 302 | where some requests to a webserver might take a long time whereas 303 | some might take a very short time. To avoid getting these quick 304 | requests stuck in the pipeline behind time consuming requests, use 305 | this function to get a handle to a connection process.
306 | Note: Calling this function only creates a worker process. No connection 307 | is setup. The connection attempt is made only when the first 308 | request is sent via any of the send_req_direct/4,5,6,7 functions.
309 | Note: It is the responsibility of the calling process to control 310 | pipeline size on such connections.

311 | 312 |

spawn_worker_process/2

313 |
314 |

spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()}

315 |

Same as spawn_worker_process/1 except with Erlang process options.

316 | 317 |

start/0

318 |
319 |

start() -> any()

320 |

Starts the ibrowse process without linking. Useful when testing using the shell

321 | 322 |

start_link/0

323 |
324 |

start_link() -> {ok, pid()}

325 |

Starts the ibrowse process linked to the calling process. Usually invoked by the supervisor ibrowse_sup

326 | 327 |

stop/0

328 |
329 |

stop() -> any()

330 |

Stop the ibrowse process. Useful when testing using the shell.

331 | 332 |

stop_worker_process/1

333 |
334 |

stop_worker_process(Conn_pid::pid()) -> ok

335 |

Terminate a worker process spawned using 336 | spawn_worker_process/2 or spawn_link_worker_process/2. Requests in 337 | progress will get the error response 

{error, closing_on_request}

338 | 339 |

stream_close/1

340 |
341 |

stream_close(Req_id::req_id()) -> ok | {error, unknown_req_id}

342 |

Tell ibrowse to close the connection associated with the 343 | specified stream. Should be used in conjunction with the 344 | stream_to option. Note that all requests in progress on 345 | the connection which is serving this Req_id will be aborted, and an 346 | error returned.

347 | 348 |

stream_next/1

349 |
350 |

stream_next(Req_id::req_id()) -> ok | {error, unknown_req_id}

351 |

Tell ibrowse to stream the next chunk of data to the 352 | caller. Should be used in conjunction with the 353 | stream_to option

354 | 355 |

terminate/2

356 |
357 |

terminate(Reason, State) -> any()

358 |
359 | 360 |

trace_off/0

361 |
362 |

trace_off() -> any()

363 |

Turn tracing off for the ibrowse process

364 | 365 |

trace_off/2

366 |
367 |

trace_off(Host, Port) -> ok

368 |

Turn tracing OFF for all connections to the specified HTTP 369 | server.

370 | 371 |

trace_on/0

372 |
373 |

trace_on() -> any()

374 |

Turn tracing on for the ibrowse process

375 | 376 |

trace_on/2

377 |
378 |

trace_on(Host, Port) -> ok 379 |

380 |

Turn tracing on for all connections to the specified HTTP 381 | server. Host is whatever is specified as the domain name in the URL

382 |
383 | 384 |
Overviewerlang logo
385 |

Generated by EDoc, Nov 6 2015, 11:40:24.

386 | 387 | 388 | -------------------------------------------------------------------------------- /doc/ibrowse_app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse_app 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse_app

13 | 14 | 15 |

Behaviours: application.

16 | 17 |

Function Index

18 | 19 | 20 |
start/2
stop/1
21 | 22 |

Function Details

23 | 24 |

start/2

25 |
26 |

start(Type, StartArgs) -> any()

27 |
28 | 29 |

stop/1

30 |
31 |

stop(State) -> any()

32 |
33 |
34 | 35 |
Overviewerlang logo
36 |

Generated by EDoc, Nov 6 2015, 11:40:24.

37 | 38 | 39 | -------------------------------------------------------------------------------- /doc/ibrowse_http_client.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse_http_client 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse_http_client

13 | 14 | 15 |

Behaviours: gen_server.

16 | 17 |

Function Index

18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
code_change/3
handle_call/3
handle_cast/2
handle_info/2
init/1
send_req/7
start/1
start/2
start_link/1
start_link/2
stop/1
terminate/2
31 | 32 |

Function Details

33 | 34 |

code_change/3

35 |
36 |

code_change(OldVsn, State, Extra) -> any()

37 |
38 | 39 |

handle_call/3

40 |
41 |

handle_call(Request, From, State) -> any()

42 |
43 | 44 |

handle_cast/2

45 |
46 |

handle_cast(Msg, State) -> any()

47 |
48 | 49 |

handle_info/2

50 |
51 |

handle_info(Info, State) -> any()

52 |
53 | 54 |

init/1

55 |
56 |

init(Url) -> any()

57 |
58 | 59 |

send_req/7

60 |
61 |

send_req(Conn_Pid, Url, Headers, Method, Body, Options, Timeout) -> any()

62 |
63 | 64 |

start/1

65 |
66 |

start(Args) -> any()

67 |
68 | 69 |

start/2

70 |
71 |

start(Args, Options) -> any()

72 |
73 | 74 |

start_link/1

75 |
76 |

start_link(Args) -> any()

77 |
78 | 79 |

start_link/2

80 |
81 |

start_link(Args, Options) -> any()

82 |
83 | 84 |

stop/1

85 |
86 |

stop(Conn_pid) -> any()

87 |
88 | 89 |

terminate/2

90 |
91 |

terminate(Reason, State) -> any()

92 |
93 |
94 | 95 |
Overviewerlang logo
96 |

Generated by EDoc, Nov 6 2015, 11:40:24.

97 | 98 | 99 | -------------------------------------------------------------------------------- /doc/ibrowse_lb.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse_lb 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse_lb

13 | 14 | 15 |

Behaviours: gen_server.

16 | 17 |

Function Index

18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 |
code_change/3
handle_call/3
handle_cast/2
handle_info/2
init/1
spawn_connection/6
start_link/1
stop/1
terminate/2
28 | 29 |

Function Details

30 | 31 |

code_change/3

32 |
33 |

code_change(OldVsn, State, Extra) -> any()

34 |
35 | 36 |

handle_call/3

37 |
38 |

handle_call(Request, From, State) -> any()

39 |
40 | 41 |

handle_cast/2

42 |
43 |

handle_cast(Msg, State) -> any()

44 |
45 | 46 |

handle_info/2

47 |
48 |

handle_info(Info, State) -> any()

49 |
50 | 51 |

init/1

52 |
53 |

init(X1) -> any()

54 |
55 | 56 |

spawn_connection/6

57 |
58 |

spawn_connection(Lb_pid, Url, Max_sessions, Max_pipeline_size, SSL_options, Process_options) -> any()

59 |
60 | 61 |

start_link/1

62 |
63 |

start_link(Args) -> any()

64 |
65 | 66 |

stop/1

67 |
68 |

stop(Lb_pid) -> any()

69 |
70 | 71 |

terminate/2

72 |
73 |

terminate(Reason, State) -> any()

74 |
75 |
76 | 77 |
Overviewerlang logo
78 |

Generated by EDoc, Nov 6 2015, 11:40:24.

79 | 80 | 81 | -------------------------------------------------------------------------------- /doc/ibrowse_lib.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse_lib 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse_lib

13 | Module with a few useful functions. 14 | 15 | 16 |

Description

Module with a few useful functions 17 |

Function Index

18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
decode_base64/1Implements the base64 decoding algorithm.
decode_rfc822_date/1
do_trace/2
do_trace/3
do_trace/3
encode_base64/1Implements the base64 encoding algorithm.
get_trace_status/2
get_value/2
get_value/3
parse_url/1
printable_date/0
printable_date/1
status_code/1Given a status code, returns an atom describing the status code.
url_encode/1URL-encodes a string based on RFC 1738.
33 | 34 |

Function Details

35 | 36 |

decode_base64/1

37 |
38 |

decode_base64(List::In) -> Out | exit({error, invalid_input}) 39 |

40 |

Implements the base64 decoding algorithm. The output data type matches in the input data type.

41 | 42 |

decode_rfc822_date/1

43 |
44 |

decode_rfc822_date(String) -> any()

45 |
46 | 47 |

do_trace/2

48 |
49 |

do_trace(Fmt, Args) -> any()

50 |
51 | 52 |

do_trace/3

53 |
54 |

do_trace(X1, Fmt, Args) -> any()

55 |
56 | 57 |

do_trace/3

58 |
59 |

do_trace(X1, Fmt, Args) -> any()

60 |
61 | 62 |

encode_base64/1

63 |
64 |

encode_base64(List::In) -> Out 65 |

66 |

Implements the base64 encoding algorithm. The output data type matches in the input data type.

67 | 68 |

get_trace_status/2

69 |
70 |

get_trace_status(Host, Port) -> any()

71 |
72 | 73 |

get_value/2

74 |
75 |

get_value(Tag, TVL) -> any()

76 |
77 | 78 |

get_value/3

79 |
80 |

get_value(Tag, TVL, DefVal) -> any()

81 |
82 | 83 |

parse_url/1

84 |
85 |

parse_url(Url) -> any()

86 |
87 | 88 |

printable_date/0

89 |
90 |

printable_date() -> any()

91 |
92 | 93 |

printable_date/1

94 |
95 |

printable_date(Now) -> any()

96 |
97 | 98 |

status_code/1

99 |
100 |

status_code(StatusCode::status_code()) -> StatusDescription 101 |

102 |

Given a status code, returns an atom describing the status code.

103 | 104 |

url_encode/1

105 |
106 |

url_encode(Str) -> UrlEncodedStr 107 |

108 |

URL-encodes a string based on RFC 1738. Returns a flat list.

109 |
110 | 111 |
Overviewerlang logo
112 |

Generated by EDoc, Nov 6 2015, 11:40:24.

113 | 114 | 115 | -------------------------------------------------------------------------------- /doc/ibrowse_socks5.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse_socks5 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse_socks5

13 | 14 | 15 | 16 |

Function Index

17 | 18 |
connect/5
19 | 20 |

Function Details

21 | 22 |

connect/5

23 |
24 |

connect(Host, Port, Options, SockOptions, Timeout) -> any()

25 |
26 |
27 | 28 |
Overviewerlang logo
29 |

Generated by EDoc, Nov 6 2015, 11:40:24.

30 | 31 | 32 | -------------------------------------------------------------------------------- /doc/ibrowse_sup.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Module ibrowse_sup 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |
11 | 12 |

Module ibrowse_sup

13 | 14 | 15 |

Behaviours: supervisor.

16 | 17 |

Function Index

18 | 19 | 20 |
init/1
start_link/0
21 | 22 |

Function Details

23 | 24 |

init/1

25 |
26 |

init(X1) -> any()

27 |
28 | 29 |

start_link/0

30 |
31 |

start_link() -> any()

32 |
33 |
34 | 35 |
Overviewerlang logo
36 |

Generated by EDoc, Nov 6 2015, 11:40:24.

37 | 38 | 39 | -------------------------------------------------------------------------------- /doc/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The ibrowse application 5 | 6 | 7 | 8 | 9 | 10 | 11 | <h2>This page uses frames</h2> 12 | <p>Your browser does not accept frames. 13 | <br>You should go to the <a href="overview-summary.html">non-frame version</a> instead. 14 | </p> 15 | 16 | 17 | -------------------------------------------------------------------------------- /doc/modules-frame.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The ibrowse application 5 | 6 | 7 | 8 |

Modules

9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
ibrowse
ibrowse_app
ibrowse_http_client
ibrowse_lb
ibrowse_lib
ibrowse_socks5
ibrowse_sup
17 | 18 | -------------------------------------------------------------------------------- /doc/overview-summary.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | The ibrowse application 6 | 7 | 8 | 9 |
Overviewerlang logo
10 |

The ibrowse application

11 | 12 |
13 |
Overviewerlang logo
14 |

Generated by EDoc, Nov 6 2015, 11:40:24.

15 | 16 | 17 | -------------------------------------------------------------------------------- /doc/short-desc: -------------------------------------------------------------------------------- 1 | A powerful HTTP/1.1 client written in erlang 2 | -------------------------------------------------------------------------------- /doc/stylesheet.css: -------------------------------------------------------------------------------- 1 | /* standard EDoc style sheet */ 2 | body { 3 | font-family: Verdana, Arial, Helvetica, sans-serif; 4 | margin-left: .25in; 5 | margin-right: .2in; 6 | margin-top: 0.2in; 7 | margin-bottom: 0.2in; 8 | color: #000000; 9 | background-color: #ffffff; 10 | } 11 | h1,h2 { 12 | margin-left: -0.2in; 13 | } 14 | div.navbar { 15 | background-color: #add8e6; 16 | padding: 0.2em; 17 | } 18 | h2.indextitle { 19 | padding: 0.4em; 20 | background-color: #add8e6; 21 | } 22 | h3.function,h3.typedecl { 23 | background-color: #add8e6; 24 | padding-left: 1em; 25 | } 26 | div.spec { 27 | margin-left: 2em; 28 | background-color: #eeeeee; 29 | } 30 | a.module { 31 | text-decoration:none 32 | } 33 | a.module:hover { 34 | background-color: #eeeeee; 35 | } 36 | ul.definitions { 37 | list-style-type: none; 38 | } 39 | ul.index { 40 | list-style-type: none; 41 | background-color: #eeeeee; 42 | } 43 | 44 | /* 45 | * Minor style tweaks 46 | */ 47 | ul { 48 | list-style-type: square; 49 | } 50 | table { 51 | border-collapse: collapse; 52 | } 53 | td { 54 | padding: 3 55 | } 56 | -------------------------------------------------------------------------------- /include/ibrowse.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(IBROWSE_HRL). 2 | -define(IBROWSE_HRL, "ibrowse.hrl"). 3 | 4 | -record(url, { 5 | abspath, 6 | host, 7 | port, 8 | username, 9 | password, 10 | path, 11 | protocol, 12 | host_type % 'hostname', 'ipv4_address' or 'ipv6_address' 13 | }). 14 | 15 | -record(lb_pid, {host_port, pid, ets_tid}). 16 | 17 | -record(client_conn, {key, cur_pipeline_size = 0, reqs_served = 0}). 18 | 19 | -record(ibrowse_conf, {key, value}). 20 | 21 | -define(CONNECTIONS_LOCAL_TABLE, ibrowse_lb). 22 | -define(LOAD_BALANCER_NAMED_TABLE, ibrowse_lb). 23 | -define(CONF_TABLE, ibrowse_conf). 24 | -define(STREAM_TABLE, ibrowse_stream). 25 | 26 | -endif. 27 | -------------------------------------------------------------------------------- /priv/ibrowse.conf: -------------------------------------------------------------------------------- 1 | %% Configuration file for specifying settings for HTTP servers which this 2 | %% client will connect to. 3 | %% The format of each entry is (one per line) 4 | %% {dest, Hostname, Portnumber, MaxSessions, MaxPipelineSize, Options}. 5 | %% 6 | %% where Hostname = string() 7 | %% Portnumber = integer() 8 | %% MaxSessions = integer() 9 | %% MaxPipelineSize = integer() 10 | %% Options = [{Tag, Val} | ...] 11 | %% Tag = term() 12 | %% Value = term() 13 | %% e.g. 14 | %% {dest, "covig02", 8000, 10, 10, [{is_ssl, true}, {ssl_options, [option()]}]}. 15 | %% If SSL is to be used, both the options, is_ssl and ssl_options MUST be specified 16 | %% where option() is all options supported in the ssl module 17 | 18 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [ 2 | debug_info, 3 | warnings_as_errors, 4 | warn_unused_vars, 5 | nowarn_shadow_vars, 6 | warn_unused_import, 7 | {platform_define, "18|19|^2", new_rand}, 8 | {platform_define, "^2", ets_ref} 9 | ]}. 10 | {xref_checks, [undefined_function_calls, deprecated_function_calls]}. 11 | {eunit_opts, [verbose]}. 12 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /src/Emakefile.src: -------------------------------------------------------------------------------- 1 | '../src/ibrowse'. 2 | '../src/ibrowse_http_client'. 3 | '../src/ibrowse_app'. 4 | '../src/ibrowse_sup'. 5 | '../src/ibrowse_lib'. 6 | '../src/ibrowse_lb'. 7 | '../src/ibrowse_test'. 8 | -------------------------------------------------------------------------------- /src/ibrowse.app.src: -------------------------------------------------------------------------------- 1 | {application, ibrowse, 2 | [{description, "Erlang HTTP client application"}, 3 | {vsn, "4.4.2"}, 4 | {registered, [ibrowse_sup, ibrowse]}, 5 | {applications, [kernel,stdlib]}, 6 | {env, []}, 7 | {mod, {ibrowse_app, []}}, 8 | {maintainers, ["Chandrashekhar Mullaparthi"]}, 9 | {licenses, ["GPLv2", "BSD"]}, 10 | {modules, []}, 11 | {links, [{"Github", "https://github.com/cmullaparthi/ibrowse"}]} 12 | ] 13 | }. 14 | -------------------------------------------------------------------------------- /src/ibrowse_app.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% File : ibrowse_app.erl 3 | %%% Author : Chandrashekhar Mullaparthi 4 | %%% Description : 5 | %%% 6 | %%% Created : 15 Oct 2003 by Chandrashekhar Mullaparthi 7 | %%%------------------------------------------------------------------- 8 | -module(ibrowse_app). 9 | 10 | -behaviour(application). 11 | %%-------------------------------------------------------------------- 12 | %% Include files 13 | %%-------------------------------------------------------------------- 14 | 15 | %%-------------------------------------------------------------------- 16 | %% External exports 17 | %%-------------------------------------------------------------------- 18 | -export([ 19 | start/2, 20 | stop/1 21 | ]). 22 | 23 | %%-------------------------------------------------------------------- 24 | %% Internal exports 25 | %%-------------------------------------------------------------------- 26 | -export([ 27 | ]). 28 | 29 | %%-------------------------------------------------------------------- 30 | %% Macros 31 | %%-------------------------------------------------------------------- 32 | 33 | %%-------------------------------------------------------------------- 34 | %% Records 35 | %%-------------------------------------------------------------------- 36 | 37 | %%==================================================================== 38 | %% External functions 39 | %%==================================================================== 40 | %%-------------------------------------------------------------------- 41 | %% Func: start/2 42 | %% Returns: {ok, Pid} | 43 | %% {ok, Pid, State} | 44 | %% {error, Reason} 45 | %%-------------------------------------------------------------------- 46 | start(_Type, _StartArgs) -> 47 | case ibrowse_sup:start_link() of 48 | {ok, Pid} -> 49 | {ok, Pid}; 50 | Error -> 51 | Error 52 | end. 53 | 54 | %%-------------------------------------------------------------------- 55 | %% Func: stop/1 56 | %% Returns: any 57 | %%-------------------------------------------------------------------- 58 | stop(_State) -> 59 | ok. 60 | 61 | %%==================================================================== 62 | %% Internal functions 63 | %%==================================================================== 64 | -------------------------------------------------------------------------------- /src/ibrowse_lb.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% File : ibrowse_lb.erl 3 | %%% Author : chandru 4 | %%% Description : 5 | %%% 6 | %%% Created : 6 Mar 2008 by chandru 7 | %%%------------------------------------------------------------------- 8 | -module(ibrowse_lb). 9 | -author(chandru). 10 | -behaviour(gen_server). 11 | %%-------------------------------------------------------------------- 12 | %% Include files 13 | %%-------------------------------------------------------------------- 14 | 15 | %%-------------------------------------------------------------------- 16 | %% External exports 17 | -export([ 18 | start_link/1, 19 | spawn_connection/6, 20 | stop/1 21 | ]). 22 | 23 | %% gen_server callbacks 24 | -export([ 25 | init/1, 26 | handle_call/3, 27 | handle_cast/2, 28 | handle_info/2, 29 | terminate/2, 30 | code_change/3 31 | ]). 32 | 33 | -record(state, {parent_pid, 34 | ets_tid, 35 | host, 36 | port, 37 | max_sessions, 38 | max_pipeline_size, 39 | proc_state 40 | }). 41 | 42 | -include("ibrowse.hrl"). 43 | 44 | %%==================================================================== 45 | %% External functions 46 | %%==================================================================== 47 | %%-------------------------------------------------------------------- 48 | %% Function: start_link/0 49 | %% Description: Starts the server 50 | %%-------------------------------------------------------------------- 51 | start_link(Args) -> 52 | gen_server:start_link(?MODULE, Args, []). 53 | 54 | %%==================================================================== 55 | %% Server functions 56 | %%==================================================================== 57 | 58 | %%-------------------------------------------------------------------- 59 | %% Function: init/1 60 | %% Description: Initiates the server 61 | %% Returns: {ok, State} | 62 | %% {ok, State, Timeout} | 63 | %% ignore | 64 | %% {stop, Reason} 65 | %%-------------------------------------------------------------------- 66 | init([Host, Port]) -> 67 | process_flag(trap_exit, true), 68 | Max_sessions = ibrowse:get_config_value({max_sessions, Host, Port}, 10), 69 | Max_pipe_sz = ibrowse:get_config_value({max_pipeline_size, Host, Port}, 10), 70 | put(my_trace_flag, ibrowse_lib:get_trace_status(Host, Port)), 71 | put(ibrowse_trace_token, ["LB: ", Host, $:, integer_to_list(Port)]), 72 | State = #state{parent_pid = whereis(ibrowse), 73 | host = Host, 74 | port = Port, 75 | max_pipeline_size = Max_pipe_sz, 76 | max_sessions = Max_sessions}, 77 | State_1 = maybe_create_ets(State), 78 | {ok, State_1}. 79 | 80 | spawn_connection(Lb_pid, Url, 81 | Max_sessions, 82 | Max_pipeline_size, 83 | SSL_options, 84 | Process_options) 85 | when is_pid(Lb_pid), 86 | is_record(Url, url), 87 | is_integer(Max_pipeline_size), 88 | is_integer(Max_sessions) -> 89 | gen_server:call(Lb_pid, 90 | {spawn_connection, Url, Max_sessions, Max_pipeline_size, SSL_options, Process_options}). 91 | 92 | stop(Lb_pid) -> 93 | case catch gen_server:call(Lb_pid, stop) of 94 | {'EXIT', {timeout, _}} -> 95 | exit(Lb_pid, kill); 96 | ok -> 97 | ok 98 | end. 99 | %%-------------------------------------------------------------------- 100 | %% Function: handle_call/3 101 | %% Description: Handling call messages 102 | %% Returns: {reply, Reply, State} | 103 | %% {reply, Reply, State, Timeout} | 104 | %% {noreply, State} | 105 | %% {noreply, State, Timeout} | 106 | %% {stop, Reason, Reply, State} | (terminate/2 is called) 107 | %% {stop, Reason, State} (terminate/2 is called) 108 | %%-------------------------------------------------------------------- 109 | 110 | handle_call(stop, _From, #state{ets_tid = undefined} = State) -> 111 | gen_server:reply(_From, ok), 112 | {stop, normal, State}; 113 | 114 | handle_call(stop, _From, #state{ets_tid = Tid} = State) -> 115 | stop_all_conn_procs(Tid), 116 | gen_server:reply(_From, ok), 117 | {stop, normal, State}; 118 | 119 | handle_call(_, _From, #state{proc_state = shutting_down} = State) -> 120 | {reply, {error, shutting_down}, State}; 121 | 122 | handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options, Process_options}, _From, 123 | State) -> 124 | State_1 = maybe_create_ets(State), 125 | Tid = State_1#state.ets_tid, 126 | Tid_size = ets:info(Tid, size), 127 | case Tid_size >= Max_sess of 128 | true -> 129 | Reply = find_best_connection(Tid, Max_pipe), 130 | {reply, Reply, State_1#state{max_sessions = Max_sess, 131 | max_pipeline_size = Max_pipe}}; 132 | false -> 133 | {ok, Pid} = ibrowse_http_client:start({Tid, Url, SSL_options}, Process_options), 134 | Ts = os:timestamp(), 135 | ets:insert(Tid, {{1, Ts, Pid}, []}), 136 | {reply, {ok, {1, Ts, Pid}}, State_1#state{max_sessions = Max_sess, 137 | max_pipeline_size = Max_pipe}} 138 | end; 139 | 140 | handle_call(Request, _From, State) -> 141 | Reply = {unknown_request, Request}, 142 | {reply, Reply, State}. 143 | 144 | %%-------------------------------------------------------------------- 145 | %% Function: handle_cast/2 146 | %% Description: Handling cast messages 147 | %% Returns: {noreply, State} | 148 | %% {noreply, State, Timeout} | 149 | %% {stop, Reason, State} (terminate/2 is called) 150 | %%-------------------------------------------------------------------- 151 | handle_cast(_Msg, State) -> 152 | {noreply, State}. 153 | 154 | %%-------------------------------------------------------------------- 155 | %% Function: handle_info/2 156 | %% Description: Handling all non call/cast messages 157 | %% Returns: {noreply, State} | 158 | %% {noreply, State, Timeout} | 159 | %% {stop, Reason, State} (terminate/2 is called) 160 | %%-------------------------------------------------------------------- 161 | 162 | handle_info({trace, Bool}, #state{ets_tid = undefined} = State) -> 163 | put(my_trace_flag, Bool), 164 | {noreply, State}; 165 | 166 | handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> 167 | ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) -> 168 | catch Pid ! {trace, Bool}, 169 | Acc; 170 | (_, Acc) -> 171 | Acc 172 | end, undefined, Tid), 173 | put(my_trace_flag, Bool), 174 | {noreply, State}; 175 | 176 | handle_info(timeout, State) -> 177 | %% We can't shutdown the process immediately because a request 178 | %% might be in flight. So we first remove the entry from the 179 | %% ibrowse_lb ets table, and then shutdown a couple of seconds 180 | %% later 181 | ets:delete(ibrowse_lb, {State#state.host, State#state.port}), 182 | erlang:send_after(2000, self(), shutdown), 183 | {noreply, State#state{proc_state = shutting_down}}; 184 | 185 | handle_info(shutdown, State) -> 186 | {stop, normal, State}; 187 | 188 | handle_info(_Info, State) -> 189 | {noreply, State}. 190 | 191 | %%-------------------------------------------------------------------- 192 | %% Function: terminate/2 193 | %% Description: Shutdown the server 194 | %% Returns: any (ignored by gen_server) 195 | %%-------------------------------------------------------------------- 196 | terminate(_Reason, #state{host = Host, port = Port, ets_tid = Tid} = _State) -> 197 | catch ets:delete(ibrowse_lb, {Host, Port}), 198 | stop_all_conn_procs(Tid), 199 | ok. 200 | 201 | stop_all_conn_procs(Tid) -> 202 | ets:foldl(fun({{_, _, Pid}, _}, Acc) -> 203 | ibrowse_http_client:stop(Pid), 204 | Acc 205 | end, [], Tid). 206 | 207 | %%-------------------------------------------------------------------- 208 | %% Func: code_change/3 209 | %% Purpose: Convert process state when code is changed 210 | %% Returns: {ok, NewState} 211 | %%-------------------------------------------------------------------- 212 | code_change(_OldVsn, State, _Extra) -> 213 | {ok, State}. 214 | 215 | %%-------------------------------------------------------------------- 216 | %%% Internal functions 217 | %%-------------------------------------------------------------------- 218 | find_best_connection(Tid, Max_pipe) -> 219 | case ets:first(Tid) of 220 | {Spec_size, Ts, Pid} = First when Spec_size < Max_pipe -> 221 | ets:delete(Tid, First), 222 | ets:insert(Tid, {{Spec_size + 1, Ts, Pid}, []}), 223 | {ok, First}; 224 | _ -> 225 | {error, retry_later} 226 | end. 227 | 228 | maybe_create_ets(#state{ets_tid = undefined, host = Host, port = Port} = State) -> 229 | Tid = ets:new(ibrowse_lb, [public, ordered_set]), 230 | ets:insert(ibrowse_lb, #lb_pid{host_port = {Host, Port}, pid = self(), ets_tid = Tid}), 231 | State#state{ets_tid = Tid}; 232 | maybe_create_ets(State) -> 233 | State. 234 | -------------------------------------------------------------------------------- /src/ibrowse_lib.erl: -------------------------------------------------------------------------------- 1 | %%% File : ibrowse_lib.erl 2 | %%% Author : Chandrashekhar Mullaparthi 3 | %%% Description : 4 | %%% Created : 27 Feb 2004 by Chandrashekhar Mullaparthi 5 | %% @doc Module with a few useful functions 6 | 7 | -module(ibrowse_lib). 8 | -author('chandru'). 9 | -ifdef(debug). 10 | -compile(export_all). 11 | -endif. 12 | 13 | -include("ibrowse.hrl"). 14 | 15 | -ifdef(EUNIT). 16 | -include_lib("eunit/include/eunit.hrl"). 17 | -endif. 18 | 19 | -export([ 20 | get_trace_status/2, 21 | do_trace/2, 22 | do_trace/3, 23 | log_msg/2, 24 | url_encode/1, 25 | decode_rfc822_date/1, 26 | status_code/1, 27 | encode_base64/1, 28 | decode_base64/1, 29 | get_value/2, 30 | get_value/3, 31 | parse_url/1, 32 | printable_date/0, 33 | printable_date/1 34 | ]). 35 | 36 | get_trace_status(Host, Port) -> 37 | ibrowse:get_config_value({trace, Host, Port}, false). 38 | 39 | %% @doc URL-encodes a string based on RFC 1738. Returns a flat list. 40 | %% @spec url_encode(Str) -> UrlEncodedStr 41 | %% Str = string() 42 | %% UrlEncodedStr = string() 43 | url_encode(Str) when is_list(Str) -> 44 | url_encode_char(lists:reverse(Str), []). 45 | 46 | url_encode_char([X | T], Acc) when X >= $0, X =< $9 -> 47 | url_encode_char(T, [X | Acc]); 48 | url_encode_char([X | T], Acc) when X >= $a, X =< $z -> 49 | url_encode_char(T, [X | Acc]); 50 | url_encode_char([X | T], Acc) when X >= $A, X =< $Z -> 51 | url_encode_char(T, [X | Acc]); 52 | url_encode_char([X | T], Acc) when X == $-; X == $_; X == $. -> 53 | url_encode_char(T, [X | Acc]); 54 | url_encode_char([32 | T], Acc) -> 55 | url_encode_char(T, [$+ | Acc]); 56 | url_encode_char([X | T], Acc) -> 57 | url_encode_char(T, [$%, d2h(X bsr 4), d2h(X band 16#0f) | Acc]); 58 | url_encode_char([], Acc) -> 59 | Acc. 60 | 61 | d2h(N) when N<10 -> N+$0; 62 | d2h(N) -> N+$a-10. 63 | 64 | decode_rfc822_date(String) when is_list(String) -> 65 | case catch decode_rfc822_date_1(string:tokens(String, ", \t\r\n")) of 66 | {'EXIT', _} -> 67 | {error, invalid_date}; 68 | Res -> 69 | Res 70 | end. 71 | 72 | % TODO: Have to handle the Zone 73 | decode_rfc822_date_1([_,DayInt,Month,Year, Time,Zone]) -> 74 | decode_rfc822_date_1([DayInt,Month,Year, Time,Zone]); 75 | decode_rfc822_date_1([Day,Month,Year, Time,_Zone]) -> 76 | DayI = list_to_integer(Day), 77 | MonthI = month_int(Month), 78 | YearI = list_to_integer(Year), 79 | TimeTup = case string:tokens(Time, ":") of 80 | [H,M] -> 81 | {list_to_integer(H), 82 | list_to_integer(M), 83 | 0}; 84 | [H,M,S] -> 85 | {list_to_integer(H), 86 | list_to_integer(M), 87 | list_to_integer(S)} 88 | end, 89 | {{YearI,MonthI,DayI}, TimeTup}. 90 | 91 | month_int("Jan") -> 1; 92 | month_int("Feb") -> 2; 93 | month_int("Mar") -> 3; 94 | month_int("Apr") -> 4; 95 | month_int("May") -> 5; 96 | month_int("Jun") -> 6; 97 | month_int("Jul") -> 7; 98 | month_int("Aug") -> 8; 99 | month_int("Sep") -> 9; 100 | month_int("Oct") -> 10; 101 | month_int("Nov") -> 11; 102 | month_int("Dec") -> 12. 103 | 104 | %% @doc Given a status code, returns an atom describing the status code. 105 | %% @spec status_code(StatusCode::status_code()) -> StatusDescription 106 | %% status_code() = string() | integer() 107 | %% StatusDescription = atom() 108 | status_code(100) -> continue; 109 | status_code(101) -> switching_protocols; 110 | status_code(102) -> processing; 111 | status_code(200) -> ok; 112 | status_code(201) -> created; 113 | status_code(202) -> accepted; 114 | status_code(203) -> non_authoritative_information; 115 | status_code(204) -> no_content; 116 | status_code(205) -> reset_content; 117 | status_code(206) -> partial_content; 118 | status_code(207) -> multi_status; 119 | status_code(300) -> multiple_choices; 120 | status_code(301) -> moved_permanently; 121 | status_code(302) -> found; 122 | status_code(303) -> see_other; 123 | status_code(304) -> not_modified; 124 | status_code(305) -> use_proxy; 125 | status_code(306) -> unused; 126 | status_code(307) -> temporary_redirect; 127 | status_code(400) -> bad_request; 128 | status_code(401) -> unauthorized; 129 | status_code(402) -> payment_required; 130 | status_code(403) -> forbidden; 131 | status_code(404) -> not_found; 132 | status_code(405) -> method_not_allowed; 133 | status_code(406) -> not_acceptable; 134 | status_code(407) -> proxy_authentication_required; 135 | status_code(408) -> request_timeout; 136 | status_code(409) -> conflict; 137 | status_code(410) -> gone; 138 | status_code(411) -> length_required; 139 | status_code(412) -> precondition_failed; 140 | status_code(413) -> request_entity_too_large; 141 | status_code(414) -> request_uri_too_long; 142 | status_code(415) -> unsupported_media_type; 143 | status_code(416) -> requested_range_not_satisfiable; 144 | status_code(417) -> expectation_failed; 145 | status_code(422) -> unprocessable_entity; 146 | status_code(423) -> locked; 147 | status_code(424) -> failed_dependency; 148 | status_code(500) -> internal_server_error; 149 | status_code(501) -> not_implemented; 150 | status_code(502) -> bad_gateway; 151 | status_code(503) -> service_unavailable; 152 | status_code(504) -> gateway_timeout; 153 | status_code(505) -> http_version_not_supported; 154 | status_code(507) -> insufficient_storage; 155 | status_code(X) when is_list(X) -> status_code(list_to_integer(X)); 156 | status_code(_) -> unknown_status_code. 157 | 158 | %% @doc Implements the base64 encoding algorithm. The output data type matches in the input data type. 159 | %% @spec encode_base64(In) -> Out 160 | %% In = string() | binary() 161 | %% Out = string() | binary() 162 | encode_base64(List) when is_list(List) -> 163 | binary_to_list(base64:encode(List)); 164 | encode_base64(Bin) when is_binary(Bin) -> 165 | base64:encode(Bin). 166 | 167 | %% @doc Implements the base64 decoding algorithm. The output data type matches in the input data type. 168 | %% @spec decode_base64(In) -> Out | exit({error, invalid_input}) 169 | %% In = string() | binary() 170 | %% Out = string() | binary() 171 | decode_base64(List) when is_list(List) -> 172 | binary_to_list(base64:decode(List)); 173 | decode_base64(Bin) when is_binary(Bin) -> 174 | base64:decode(Bin). 175 | 176 | get_value(Tag, TVL, DefVal) -> 177 | case lists:keysearch(Tag, 1, TVL) of 178 | false -> 179 | DefVal; 180 | {value, {_, Val}} -> 181 | Val 182 | end. 183 | 184 | get_value(Tag, TVL) -> 185 | {value, {_, V}} = lists:keysearch(Tag,1,TVL), 186 | V. 187 | 188 | parse_url(Url) -> 189 | try 190 | case parse_url(Url, get_protocol, #url{abspath=Url}, []) of 191 | #url{host_type = undefined, host = Host} = UrlRec -> 192 | case inet_parse:address(Host) of 193 | {ok, {_, _, _, _, _, _, _, _}} -> 194 | UrlRec#url{host_type = ipv6_address}; 195 | {ok, {_, _, _, _}} -> 196 | UrlRec#url{host_type = ipv4_address}; 197 | _ -> 198 | UrlRec#url{host_type = hostname} 199 | end; 200 | #url{} = UrlRec -> 201 | UrlRec; 202 | _ -> 203 | {error, invalid_uri} 204 | end 205 | catch _:_ -> 206 | {error, invalid_uri} 207 | end. 208 | 209 | parse_url([$:, $/, $/ | _], get_protocol, Url, []) -> 210 | {invalid_uri_1, Url}; 211 | parse_url([$:, $/, $/ | T], get_protocol, Url, TmpAcc) -> 212 | Prot = list_to_existing_atom(lists:reverse(TmpAcc)), 213 | parse_url(T, get_username, 214 | Url#url{protocol = Prot}, 215 | []); 216 | parse_url([H | T], get_username, Url, TmpAcc) when H == $/; 217 | H == $? -> 218 | Path = case H of 219 | $/ -> 220 | [$/ | T]; 221 | $? -> 222 | [$/, $? | T] 223 | end, 224 | %% No username/password. No port number 225 | Url#url{host = lists:reverse(TmpAcc), 226 | port = default_port(Url#url.protocol), 227 | path = Path}; 228 | parse_url([$: | T], get_username, Url, TmpAcc) -> 229 | %% It is possible that no username/password has been 230 | %% specified. But we'll continue with the assumption that there is 231 | %% a username/password. If we encounter a '@' later on, there is a 232 | %% username/password indeed. If we encounter a '/', it was 233 | %% actually the hostname 234 | parse_url(T, get_password, 235 | Url#url{username = lists:reverse(TmpAcc)}, 236 | []); 237 | parse_url([$@ | T], get_username, Url, TmpAcc) -> 238 | parse_url(T, get_host, 239 | Url#url{username = lists:reverse(TmpAcc), 240 | password = ""}, 241 | []); 242 | parse_url([$[ | T], get_username, Url, []) -> 243 | % IPv6 address literals are enclosed by square brackets: 244 | % http://www.ietf.org/rfc/rfc2732.txt 245 | parse_url(T, get_ipv6_address, Url#url{host_type = ipv6_address}, []); 246 | parse_url([$[ | T], get_username, _Url, TmpAcc) -> 247 | {error, {invalid_username_or_host, lists:reverse(TmpAcc) ++ "[" ++ T}}; 248 | parse_url([$[ | _], get_password, _Url, []) -> 249 | {error, missing_password}; 250 | parse_url([$[ | T], get_password, Url, TmpAcc) -> 251 | % IPv6 address literals are enclosed by square brackets: 252 | % http://www.ietf.org/rfc/rfc2732.txt 253 | parse_url(T, get_ipv6_address, 254 | Url#url{host_type = ipv6_address, 255 | password = lists:reverse(TmpAcc)}, 256 | []); 257 | parse_url([$@ | T], get_password, Url, TmpAcc) -> 258 | parse_url(T, get_host, 259 | Url#url{password = lists:reverse(TmpAcc)}, 260 | []); 261 | parse_url([H | T], get_password, Url, TmpAcc) when H == $/; 262 | H == $? -> 263 | %% Ok, what we thought was the username/password was the hostname 264 | %% and portnumber 265 | #url{username=User} = Url, 266 | Port = list_to_integer(lists:reverse(TmpAcc)), 267 | Path = case H of 268 | $/ -> 269 | [$/ | T]; 270 | $? -> 271 | [$/, $? | T] 272 | end, 273 | Url#url{host = User, 274 | port = Port, 275 | username = undefined, 276 | password = undefined, 277 | path = Path}; 278 | parse_url([$] | T], get_ipv6_address, #url{protocol = Prot} = Url, TmpAcc) -> 279 | Addr = lists:reverse(TmpAcc), 280 | case inet_parse:address(Addr) of 281 | {ok, {_, _, _, _, _, _, _, _}} -> 282 | Url2 = Url#url{host = Addr, port = default_port(Prot)}, 283 | case T of 284 | [$: | T2] -> 285 | parse_url(T2, get_port, Url2, []); 286 | [$/ | T2] -> 287 | Url2#url{path = [$/ | T2]}; 288 | [$? | T2] -> 289 | Url2#url{path = [$/, $? | T2]}; 290 | [] -> 291 | Url2#url{path = "/"}; 292 | _ -> 293 | {error, {invalid_host, "[" ++ Addr ++ "]" ++ T}} 294 | end; 295 | _ -> 296 | {error, {invalid_ipv6_address, Addr}} 297 | end; 298 | parse_url([$[ | T], get_host, #url{} = Url, []) -> 299 | parse_url(T, get_ipv6_address, Url#url{host_type = ipv6_address}, []); 300 | parse_url([$: | T], get_host, #url{} = Url, TmpAcc) -> 301 | parse_url(T, get_port, 302 | Url#url{host = lists:reverse(TmpAcc)}, 303 | []); 304 | parse_url([H | T], get_host, #url{protocol=Prot} = Url, TmpAcc) when H == $/; 305 | H == $? -> 306 | Path = case H of 307 | $/ -> 308 | [$/ | T]; 309 | $? -> 310 | [$/, $? | T] 311 | end, 312 | Url#url{host = lists:reverse(TmpAcc), 313 | port = default_port(Prot), 314 | path = Path}; 315 | parse_url([H | T], get_port, #url{protocol=Prot} = Url, TmpAcc) when H == $/; 316 | H == $? -> 317 | Path = case H of 318 | $/ -> 319 | [$/ | T]; 320 | $? -> 321 | [$/, $? | T] 322 | end, 323 | Port = case TmpAcc of 324 | [] -> 325 | default_port(Prot); 326 | _ -> 327 | list_to_integer(lists:reverse(TmpAcc)) 328 | end, 329 | Url#url{port = Port, path = Path}; 330 | parse_url([H | T], State, Url, TmpAcc) -> 331 | parse_url(T, State, Url, [H | TmpAcc]); 332 | parse_url([], get_host, Url, TmpAcc) when TmpAcc /= [] -> 333 | Url#url{host = lists:reverse(TmpAcc), 334 | port = default_port(Url#url.protocol), 335 | path = "/"}; 336 | parse_url([], get_username, Url, TmpAcc) when TmpAcc /= [] -> 337 | Url#url{host = lists:reverse(TmpAcc), 338 | port = default_port(Url#url.protocol), 339 | path = "/"}; 340 | parse_url([], get_port, #url{protocol=Prot} = Url, TmpAcc) -> 341 | Port = case TmpAcc of 342 | [] -> 343 | default_port(Prot); 344 | _ -> 345 | list_to_integer(lists:reverse(TmpAcc)) 346 | end, 347 | Url#url{port = Port, 348 | path = "/"}; 349 | parse_url([], get_password, Url, TmpAcc) -> 350 | %% Ok, what we thought was the username/password was the hostname 351 | %% and portnumber 352 | #url{username=User} = Url, 353 | Port = case TmpAcc of 354 | [] -> 355 | default_port(Url#url.protocol); 356 | _ -> 357 | list_to_integer(lists:reverse(TmpAcc)) 358 | end, 359 | Url#url{host = User, 360 | port = Port, 361 | username = undefined, 362 | password = undefined, 363 | path = "/"}; 364 | parse_url([], State, Url, TmpAcc) -> 365 | {invalid_uri_2, State, Url, TmpAcc}. 366 | 367 | default_port(socks5) -> 1080; 368 | default_port(http) -> 80; 369 | default_port(https) -> 443; 370 | default_port(ftp) -> 21. 371 | 372 | printable_date() -> 373 | printable_date(os:timestamp()). 374 | 375 | printable_date(Now) -> 376 | {{Y,Mo,D},{H, M, S}} = calendar:now_to_local_time(Now), 377 | {_,_,MicroSecs} = Now, 378 | [integer_to_list(Y), 379 | $-, 380 | integer_to_list(Mo), 381 | $-, 382 | integer_to_list(D), 383 | $_, 384 | integer_to_list(H), 385 | $:, 386 | integer_to_list(M), 387 | $:, 388 | integer_to_list(S), 389 | $:, 390 | integer_to_list(MicroSecs div 1000)]. 391 | 392 | do_trace(Fmt, Args) -> 393 | do_trace(get(my_trace_flag), Fmt, Args). 394 | 395 | -ifdef(DEBUG). 396 | do_trace(_, Fmt, Args) -> 397 | io:format("~s -- (~s) - "++Fmt, 398 | [printable_date(), 399 | get(ibrowse_trace_token) | Args]). 400 | -else. 401 | do_trace(true, Fmt, Args) -> 402 | Fmt_1 = "~s -- (~s) - "++Fmt, 403 | Args_1 = [printable_date(), 404 | get(ibrowse_trace_token) | Args], 405 | case application:get_env(ibrowse, logger_mf) of 406 | {ok, {M, F}} -> 407 | log_msg(M, F, Fmt_1, Args_1); 408 | _ -> 409 | log_msg(io, format, Fmt_1, Args_1) 410 | end; 411 | do_trace(_, _, _) -> 412 | ok. 413 | -endif. 414 | 415 | log_msg(Fmt, Args) -> 416 | case application:get_env(ibrowse, logger_mf) of 417 | {ok, {M, F}} -> 418 | log_msg(M, F, Fmt, Args), 419 | ok; 420 | _ -> 421 | ok 422 | end. 423 | 424 | log_msg(M, F, Fmt, Args) -> 425 | catch apply(M, F, [Fmt, Args]). 426 | 427 | -ifdef(EUNIT). 428 | 429 | parse_url_test() -> 430 | Urls = [{"http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80/index.html", 431 | #url{abspath = "http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80/index.html", 432 | host = "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210", 433 | port = 80, protocol = http, path = "/index.html", 434 | host_type = ipv6_address}}, 435 | {"http://[1080:0:0:0:8:800:200C:417A]/index.html", 436 | #url{abspath = "http://[1080:0:0:0:8:800:200C:417A]/index.html", 437 | host_type = ipv6_address, port = 80, protocol = http, 438 | host = "1080:0:0:0:8:800:200C:417A", path = "/index.html"}}, 439 | {"http://[3ffe:2a00:100:7031::1]", 440 | #url{abspath = "http://[3ffe:2a00:100:7031::1]", 441 | host_type = ipv6_address, port = 80, protocol = http, 442 | host = "3ffe:2a00:100:7031::1", path = "/"}}, 443 | {"http://[1080::8:800:200C:417A]/foo", 444 | #url{abspath = "http://[1080::8:800:200C:417A]/foo", 445 | host_type = ipv6_address, port = 80, protocol = http, 446 | host = "1080::8:800:200C:417A", path = "/foo"}}, 447 | {"http://[::192.9.5.5]/ipng", 448 | #url{abspath = "http://[::192.9.5.5]/ipng", 449 | host_type = ipv6_address, port = 80, protocol = http, 450 | host = "::192.9.5.5", path = "/ipng"}}, 451 | {"http://[::FFFF:129.144.52.38]:80/index.html", 452 | #url{abspath = "http://[::FFFF:129.144.52.38]:80/index.html", 453 | host_type = ipv6_address, port = 80, protocol = http, 454 | host = "::FFFF:129.144.52.38", path = "/index.html"}}, 455 | {"http://[2010:836B:4179::836B:4179]", 456 | #url{abspath = "http://[2010:836B:4179::836B:4179]", 457 | host_type = ipv6_address, port = 80, protocol = http, 458 | host = "2010:836B:4179::836B:4179", path = "/"}} 459 | ], 460 | lists:foreach( 461 | fun({Url, Expected_result}) -> 462 | ?assertMatch(Expected_result, parse_url(Url)) 463 | end, Urls). 464 | 465 | -endif. 466 | -------------------------------------------------------------------------------- /src/ibrowse_socks5.erl: -------------------------------------------------------------------------------- 1 | % Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | % use this file except in compliance with the License. You may obtain a copy of 3 | % the License at 4 | % 5 | % http://www.apache.org/licenses/LICENSE-2.0 6 | % 7 | % Unless required by applicable law or agreed to in writing, software 8 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | % License for the specific language governing permissions and limitations under 11 | % the License. 12 | 13 | -module(ibrowse_socks5). 14 | 15 | -define(VERSION, 5). 16 | -define(CONNECT, 1). 17 | 18 | -define(NO_AUTH, 0). 19 | -define(USERPASS, 2). 20 | -define(UNACCEPTABLE, 16#FF). 21 | -define(RESERVED, 0). 22 | 23 | -define(ATYP_IPV4, 1). 24 | -define(ATYP_DOMAINNAME, 3). 25 | -define(ATYP_IPV6, 4). 26 | 27 | -define(SUCCEEDED, 0). 28 | 29 | -export([connect/5]). 30 | 31 | -import(ibrowse_lib, [get_value/2, get_value/3]). 32 | 33 | connect(Host, Port, Options, SockOptions, Timeout) -> 34 | Socks5Host = get_value(socks5_host, Options), 35 | Socks5Port = get_value(socks5_port, Options), 36 | case gen_tcp:connect(Socks5Host, Socks5Port, SockOptions, Timeout) of 37 | {ok, Socket} -> 38 | case handshake(Socket, Options) of 39 | ok -> 40 | case connect(Host, Port, Socket) of 41 | ok -> 42 | {ok, Socket}; 43 | Else -> 44 | gen_tcp:close(Socket), 45 | Else 46 | end; 47 | Else -> 48 | gen_tcp:close(Socket), 49 | Else 50 | end; 51 | Else -> 52 | Else 53 | end. 54 | 55 | handshake(Socket, Options) when is_port(Socket) -> 56 | User = get_value(socks5_user, Options, <<>>), 57 | Handshake_msg = case User of 58 | <<>> -> 59 | <>; 60 | User -> 61 | <> 62 | end, 63 | ok = gen_tcp:send(Socket, Handshake_msg), 64 | case gen_tcp:recv(Socket, 2) of 65 | {ok, <>} -> 66 | ok; 67 | {ok, <>} -> 68 | Password = get_value(socks5_password, Options, <<>>), 69 | Auth_msg = list_to_binary([1, 70 | iolist_size(User), User, 71 | iolist_size(Password), Password]), 72 | ok = gen_tcp:send(Socket, Auth_msg), 73 | case gen_tcp:recv(Socket, 2) of 74 | {ok, <<1, ?SUCCEEDED>>} -> 75 | ok; 76 | _ -> 77 | {error, unacceptable} 78 | end; 79 | {ok, <>} -> 80 | {error, unacceptable}; 81 | {error, Reason} -> 82 | {error, Reason} 83 | end. 84 | 85 | connect(Host, Port, Via) when is_list(Host) -> 86 | connect(list_to_binary(Host), Port, Via); 87 | connect(Host, Port, Via) when is_binary(Host), is_integer(Port), 88 | is_port(Via) -> 89 | {AddressType, Address} = case inet:parse_address(binary_to_list(Host)) of 90 | {ok, {IP1, IP2, IP3, IP4}} -> 91 | {?ATYP_IPV4, <>}; 92 | {ok, {IP1, IP2, IP3, IP4, IP5, IP6, IP7, IP8}} -> 93 | {?ATYP_IPV6, <>}; 94 | _ -> 95 | HostLength = byte_size(Host), 96 | {?ATYP_DOMAINNAME, <>} 97 | end, 98 | ok = gen_tcp:send(Via, 99 | <>), 102 | case gen_tcp:recv(Via, 0) of 103 | {ok, <>} -> 104 | ok; 105 | {ok, <>} -> 106 | {error, rep(Rep)}; 107 | {error, Reason} -> 108 | {error, Reason} 109 | end. 110 | 111 | rep(0) -> succeeded; 112 | rep(1) -> server_fail; 113 | rep(2) -> disallowed_by_ruleset; 114 | rep(3) -> network_unreachable; 115 | rep(4) -> host_unreachable; 116 | rep(5) -> connection_refused; 117 | rep(6) -> ttl_expired; 118 | rep(7) -> command_not_supported; 119 | rep(8) -> address_type_not_supported. 120 | -------------------------------------------------------------------------------- /src/ibrowse_sup.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% File : ibrowse_sup.erl 3 | %%% Author : Chandrashekhar Mullaparthi 4 | %%% Description : 5 | %%% 6 | %%% Created : 15 Oct 2003 by Chandrashekhar Mullaparthi 7 | %%%------------------------------------------------------------------- 8 | -module(ibrowse_sup). 9 | -behaviour(supervisor). 10 | %%-------------------------------------------------------------------- 11 | %% Include files 12 | %%-------------------------------------------------------------------- 13 | 14 | %%-------------------------------------------------------------------- 15 | %% External exports 16 | %%-------------------------------------------------------------------- 17 | -export([ 18 | start_link/0 19 | ]). 20 | 21 | %%-------------------------------------------------------------------- 22 | %% Internal exports 23 | %%-------------------------------------------------------------------- 24 | -export([ 25 | init/1 26 | ]). 27 | 28 | %%-------------------------------------------------------------------- 29 | %% Macros 30 | %%-------------------------------------------------------------------- 31 | -define(SERVER, ?MODULE). 32 | 33 | %%-------------------------------------------------------------------- 34 | %% Records 35 | %%-------------------------------------------------------------------- 36 | 37 | %%==================================================================== 38 | %% External functions 39 | %%==================================================================== 40 | %%-------------------------------------------------------------------- 41 | %% Function: start_link/0 42 | %% Description: Starts the supervisor 43 | %%-------------------------------------------------------------------- 44 | start_link() -> 45 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 46 | 47 | %%==================================================================== 48 | %% Server functions 49 | %%==================================================================== 50 | %%-------------------------------------------------------------------- 51 | %% Func: init/1 52 | %% Returns: {ok, {SupFlags, [ChildSpec]}} | 53 | %% ignore | 54 | %% {error, Reason} 55 | %%-------------------------------------------------------------------- 56 | init([]) -> 57 | AChild = {ibrowse,{ibrowse,start_link,[]}, 58 | permanent,2000,worker,[ibrowse, ibrowse_http_client]}, 59 | {ok,{{one_for_all,10,1}, [AChild]}}. 60 | 61 | %%==================================================================== 62 | %% Internal functions 63 | %%==================================================================== 64 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | IBROWSE_EBIN_PATH=../../ibrowse/ebin 2 | 3 | compile: 4 | @erl -pa $(IBROWSE_EBIN_PATH) -make 5 | 6 | test: compile 7 | @erl -noshell -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test local_unit_tests -s erlang halt 8 | 9 | old_tests: compile 10 | @erl -noshell -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test unit_tests -s erlang halt 11 | 12 | test_shell: compile 13 | erl -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test_server start_server 14 | 15 | clean: 16 | @rm -f *.beam 17 | -------------------------------------------------------------------------------- /test/ibrowse_lib_tests.erl: -------------------------------------------------------------------------------- 1 | %%% File : ibrowse_lib.erl 2 | %%% Authors : Chandrashekhar Mullaparthi , 3 | %%% Filipe David Manana 4 | %%% Description : Tests for the module ibrowse_lib.erl 5 | %%% Created : 12 April 2011 by Filipe David Manana 6 | 7 | -module(ibrowse_lib_tests). 8 | -include_lib("eunit/include/eunit.hrl"). 9 | -include("../include/ibrowse.hrl"). 10 | 11 | 12 | parse_urls_test_() -> 13 | {timeout, 60, [fun parse_urls/0]}. 14 | 15 | 16 | parse_urls() -> 17 | ?assertMatch(#url{ 18 | abspath = "http://localhost", 19 | host = "localhost", 20 | host_type = hostname, 21 | port = 80, 22 | path = "/", 23 | username = undefined, 24 | password = undefined, 25 | protocol = http 26 | }, 27 | ibrowse_lib:parse_url("http://localhost")), 28 | ?assertMatch(#url{ 29 | abspath = "http://localhost:80/", 30 | host = "localhost", 31 | host_type = hostname, 32 | port = 80, 33 | path = "/", 34 | username = undefined, 35 | password = undefined, 36 | protocol = http 37 | }, 38 | ibrowse_lib:parse_url("http://localhost:80/")), 39 | ?assertMatch(#url{ 40 | abspath = "http://127.0.0.1:8000/", 41 | host = "127.0.0.1", 42 | host_type = ipv4_address, 43 | port = 8000, 44 | path = "/", 45 | username = undefined, 46 | password = undefined, 47 | protocol = http 48 | }, 49 | ibrowse_lib:parse_url("http://127.0.0.1:8000/")), 50 | ?assertMatch(#url{ 51 | abspath = "https://foo:bar@127.0.0.1:8000/test", 52 | host = "127.0.0.1", 53 | host_type = ipv4_address, 54 | port = 8000, 55 | path = "/test", 56 | username = "foo", 57 | password = "bar", 58 | protocol = https 59 | }, 60 | ibrowse_lib:parse_url("https://foo:bar@127.0.0.1:8000/test")), 61 | ?assertMatch(#url{ 62 | abspath = "https://[::1]", 63 | host = "::1", 64 | host_type = ipv6_address, 65 | port = 443, 66 | path = "/", 67 | username = undefined, 68 | password = undefined, 69 | protocol = https 70 | }, 71 | ibrowse_lib:parse_url("https://[::1]")), 72 | ?assertMatch(#url{ 73 | abspath = "http://[::1]:8080", 74 | host = "::1", 75 | host_type = ipv6_address, 76 | port = 8080, 77 | path = "/", 78 | username = undefined, 79 | password = undefined, 80 | protocol = http 81 | }, 82 | ibrowse_lib:parse_url("http://[::1]:8080")), 83 | ?assertMatch(#url{ 84 | abspath = "http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8081/index.html", 85 | host = "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210", 86 | host_type = ipv6_address, 87 | port = 8081, 88 | path = "/index.html", 89 | username = undefined, 90 | password = undefined, 91 | protocol = http 92 | }, 93 | ibrowse_lib:parse_url("http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8081/index.html")), 94 | ?assertMatch(#url{ 95 | abspath = "http://[1080:0:0:0:8:800:200C:417A]/foo/bar", 96 | host = "1080:0:0:0:8:800:200C:417A", 97 | host_type = ipv6_address, 98 | port = 80, 99 | path = "/foo/bar", 100 | username = undefined, 101 | password = undefined, 102 | protocol = http 103 | }, 104 | ibrowse_lib:parse_url("http://[1080:0:0:0:8:800:200C:417A]/foo/bar")), 105 | ?assertMatch(#url{ 106 | abspath = "http://[1080:0:0:0:8:800:200C:417A]:8080/foo/bar", 107 | host = "1080:0:0:0:8:800:200C:417A", 108 | host_type = ipv6_address, 109 | port = 8080, 110 | path = "/foo/bar", 111 | username = undefined, 112 | password = undefined, 113 | protocol = http 114 | }, 115 | ibrowse_lib:parse_url("http://[1080:0:0:0:8:800:200C:417A]:8080/foo/bar")), 116 | ?assertMatch(#url{ 117 | abspath = "http://[::192.9.5.5]:6000/foo?q=bar", 118 | host = "::192.9.5.5", 119 | host_type = ipv6_address, 120 | port = 6000, 121 | path = "/foo?q=bar", 122 | username = undefined, 123 | password = undefined, 124 | protocol = http 125 | }, 126 | ibrowse_lib:parse_url("http://[::192.9.5.5]:6000/foo?q=bar")), 127 | ?assertMatch({error, invalid_uri}, 128 | ibrowse_lib:parse_url("http://[:1080:0:0:0:8:800:200C:417A:]:6000/foo?q=bar")), 129 | ?assertMatch({error, invalid_uri}, 130 | ibrowse_lib:parse_url("http://[12::z]")), 131 | ?assertMatch({error, invalid_uri}, 132 | ibrowse_lib:parse_url("http://foo[1080:0:0:0:8:800:200C:417A]:6000")), 133 | ?assertMatch({error, invalid_uri}, 134 | ibrowse_lib:parse_url("http://foo:[1080:0:0:0:8:800:200C:417A]:6000")), 135 | ok. 136 | -------------------------------------------------------------------------------- /test/ibrowse_load_test.erl: -------------------------------------------------------------------------------- 1 | -module(ibrowse_load_test). 2 | %%-compile(export_all). 3 | -export([ 4 | random_seed/0, 5 | start/3, 6 | query_state/0, 7 | shutdown/0, 8 | start_1/3, 9 | calculate_timings/0, 10 | get_mmv/2, 11 | spawn_workers/2, 12 | spawn_workers/4, 13 | wait_for_workers/1, 14 | worker_loop/2, 15 | update_unknown_counter/2 16 | ]). 17 | 18 | -ifdef(new_rand). 19 | 20 | -define(RAND, rand). 21 | random_seed() -> 22 | ok. 23 | 24 | -else. 25 | 26 | -define(RAND, random). 27 | random_seed() -> 28 | random:seed(os:timestamp()). 29 | 30 | -endif. 31 | 32 | -define(ibrowse_load_test_counters, ibrowse_load_test_counters). 33 | 34 | start(Num_workers, Num_requests, Max_sess) -> 35 | proc_lib:spawn(fun() -> 36 | start_1(Num_workers, Num_requests, Max_sess) 37 | end). 38 | 39 | query_state() -> 40 | ibrowse_load_test ! query_state. 41 | 42 | shutdown() -> 43 | ibrowse_load_test ! shutdown. 44 | 45 | start_1(Num_workers, Num_requests, Max_sess) -> 46 | register(ibrowse_load_test, self()), 47 | application:start(ibrowse), 48 | application:set_env(ibrowse, inactivity_timeout, 5000), 49 | Ulimit = os:cmd("ulimit -n"), 50 | case catch list_to_integer(string:strip(Ulimit, right, $\n)) of 51 | X when is_integer(X), X > 3000 -> 52 | ok; 53 | X -> 54 | io:format("Load test not starting. {insufficient_value_for_ulimit, ~p}~n", [X]), 55 | exit({insufficient_value_for_ulimit, X}) 56 | end, 57 | ets:new(?ibrowse_load_test_counters, [named_table, public]), 58 | ets:new(ibrowse_load_timings, [named_table, public]), 59 | try 60 | ets:insert(?ibrowse_load_test_counters, [{success, 0}, 61 | {failed, 0}, 62 | {timeout, 0}, 63 | {retry_later, 0}, 64 | {one_request_only, 0} 65 | ]), 66 | ibrowse:set_max_sessions("localhost", 8081, Max_sess), 67 | Start_time = os:timestamp(), 68 | Workers = spawn_workers(Num_workers, Num_requests), 69 | erlang:send_after(1000, self(), print_diagnostics), 70 | ok = wait_for_workers(Workers), 71 | End_time = os:timestamp(), 72 | Time_in_secs = trunc(round(timer:now_diff(End_time, Start_time) / 1000000)), 73 | Req_count = Num_workers * Num_requests, 74 | [{_, Success_count}] = ets:lookup(?ibrowse_load_test_counters, success), 75 | case Success_count == Req_count of 76 | true -> 77 | io:format("Test success. All requests succeeded~n", []); 78 | false when Success_count > 0 -> 79 | io:format("Test failed. Some successes~n", []); 80 | false -> 81 | io:format("Test failed. ALL requests FAILED~n", []) 82 | end, 83 | case Time_in_secs > 0 of 84 | true -> 85 | io:format("Reqs/sec achieved : ~p~n", [trunc(round(Success_count / Time_in_secs))]); 86 | false -> 87 | ok 88 | end, 89 | io:format("Load test results:~n~p~n", [ets:tab2list(?ibrowse_load_test_counters)]), 90 | io:format("Timings: ~p~n", [calculate_timings()]) 91 | catch Err -> 92 | io:format("Err: ~p~n", [Err]) 93 | after 94 | ets:delete(?ibrowse_load_test_counters), 95 | ets:delete(ibrowse_load_timings), 96 | unregister(ibrowse_load_test) 97 | end. 98 | 99 | calculate_timings() -> 100 | {Max, Min, Mean} = get_mmv(ets:first(ibrowse_load_timings), {0, 9999999, 0}), 101 | Variance = trunc(round(ets:foldl(fun({_, X}, X_acc) -> 102 | (X - Mean)*(X-Mean) + X_acc 103 | end, 0, ibrowse_load_timings) / ets:info(ibrowse_load_timings, size))), 104 | Std_dev = trunc(round(math:sqrt(Variance))), 105 | {ok, [{max, Max}, 106 | {min, Min}, 107 | {mean, Mean}, 108 | {variance, Variance}, 109 | {standard_deviation, Std_dev}]}. 110 | 111 | get_mmv('$end_of_table', {Max, Min, Total}) -> 112 | Mean = trunc(round(Total / ets:info(ibrowse_load_timings, size))), 113 | {Max, Min, Mean}; 114 | get_mmv(Key, {Max, Min, Total}) -> 115 | [{_, V}] = ets:lookup(ibrowse_load_timings, Key), 116 | get_mmv(ets:next(ibrowse_load_timings, Key), {max(Max, V), min(Min, V), Total + V}). 117 | 118 | 119 | spawn_workers(Num_w, Num_r) -> 120 | spawn_workers(Num_w, Num_r, self(), []). 121 | 122 | spawn_workers(0, _Num_requests, _Parent, Acc) -> 123 | lists:reverse(Acc); 124 | spawn_workers(Num_workers, Num_requests, Parent, Acc) -> 125 | Pid_ref = spawn_monitor(fun() -> 126 | random_seed(), 127 | case catch worker_loop(Parent, Num_requests) of 128 | {'EXIT', Rsn} -> 129 | io:format("Worker crashed with reason: ~p~n", [Rsn]); 130 | _ -> 131 | ok 132 | end 133 | end), 134 | spawn_workers(Num_workers - 1, Num_requests, Parent, [Pid_ref | Acc]). 135 | 136 | wait_for_workers([]) -> 137 | ok; 138 | wait_for_workers([{Pid, Pid_ref} | T] = Pids) -> 139 | receive 140 | {done, Pid} -> 141 | wait_for_workers(T); 142 | {done, Some_pid} -> 143 | wait_for_workers([{Pid, Pid_ref} | lists:keydelete(Some_pid, 1, T)]); 144 | print_diagnostics -> 145 | io:format("~1000.p~n", [ibrowse:get_metrics()]), 146 | erlang:send_after(1000, self(), print_diagnostics), 147 | wait_for_workers(Pids); 148 | query_state -> 149 | io:format("Waiting for ~p~n", [Pids]), 150 | wait_for_workers(Pids); 151 | shutdown -> 152 | io:format("Shutting down on command. Still waiting for ~p workers~n", [length(Pids)]); 153 | {'DOWN', _, process, _, normal} -> 154 | wait_for_workers(Pids); 155 | {'DOWN', _, process, Down_pid, Rsn} -> 156 | io:format("Worker ~p died. Reason: ~p~n", [Down_pid, Rsn]), 157 | wait_for_workers(lists:keydelete(Down_pid, 1, Pids)); 158 | X -> 159 | io:format("Recvd unknown msg: ~p~n", [X]), 160 | wait_for_workers(Pids) 161 | end. 162 | 163 | worker_loop(Parent, 0) -> 164 | Parent ! {done, self()}; 165 | worker_loop(Parent, N) -> 166 | Delay = ?RAND:uniform(100), 167 | Url = case Delay rem 10 of 168 | %% Change 10 to some number between 0-9 depending on how 169 | %% much chaos you want to introduce into the server 170 | %% side. The higher the number, the more often the 171 | %% server will close a connection after serving the 172 | %% first request, thereby forcing the client to 173 | %% retry. Any number of 10 or higher will disable this 174 | %% chaos mechanism 175 | 10 -> 176 | ets:update_counter(?ibrowse_load_test_counters, one_request_only, 1), 177 | "http://localhost:8081/ibrowse_handle_one_request_only"; 178 | _ -> 179 | "http://localhost:8081/blah" 180 | end, 181 | Start_time = os:timestamp(), 182 | Res = ibrowse:send_req(Url, [], get), 183 | End_time = os:timestamp(), 184 | Time_taken = trunc(round(timer:now_diff(End_time, Start_time) / 1000)), 185 | ets:insert(ibrowse_load_timings, {os:timestamp(), Time_taken}), 186 | case Res of 187 | {ok, "200", _, _} -> 188 | ets:update_counter(?ibrowse_load_test_counters, success, 1); 189 | {error, req_timedout} -> 190 | ets:update_counter(?ibrowse_load_test_counters, timeout, 1); 191 | {error, retry_later} -> 192 | ets:update_counter(?ibrowse_load_test_counters, retry_later, 1); 193 | {error, Reason} -> 194 | update_unknown_counter(Reason, 1); 195 | _ -> 196 | io:format("~p -- Res: ~p~n", [self(), Res]), 197 | ets:update_counter(?ibrowse_load_test_counters, failed, 1) 198 | end, 199 | timer:sleep(Delay), 200 | worker_loop(Parent, N - 1). 201 | 202 | update_unknown_counter(Counter, Inc_val) -> 203 | case catch ets:update_counter(?ibrowse_load_test_counters, Counter, Inc_val) of 204 | {'EXIT', _} -> 205 | ets:insert_new(?ibrowse_load_test_counters, {Counter, 0}), 206 | update_unknown_counter(Counter, Inc_val); 207 | _ -> 208 | ok 209 | end. 210 | -------------------------------------------------------------------------------- /test/ibrowse_socks_server.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @author Chandru Mullaparthi <> 3 | %%% @copyright (C) 2016, Chandru Mullaparthi 4 | %%% @doc 5 | %%% 6 | %%% @end 7 | %%% Created : 19 Apr 2016 by Chandru Mullaparthi <> 8 | %%%------------------------------------------------------------------- 9 | -module(ibrowse_socks_server). 10 | 11 | -behaviour(gen_server). 12 | 13 | %% API 14 | -export([start/2, stop/1]). 15 | 16 | %% gen_server callbacks 17 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 18 | terminate/2, code_change/3]). 19 | 20 | -define(SERVER, ?MODULE). 21 | 22 | -record(state, {listen_port, listen_socket, auth_method}). 23 | 24 | -define(NO_AUTH, 0). 25 | -define(AUTH_USER_PW, 2). 26 | 27 | %%%=================================================================== 28 | %%% API 29 | %%%=================================================================== 30 | 31 | start(Port, Auth_method) -> 32 | Name = make_proc_name(Port), 33 | gen_server:start({local, Name}, ?MODULE, [Port, Auth_method], []). 34 | 35 | stop(Port) -> 36 | make_proc_name(Port) ! stop. 37 | 38 | make_proc_name(Port) -> 39 | list_to_atom("ibrowse_socks_server_" ++ integer_to_list(Port)). 40 | 41 | %%%=================================================================== 42 | %%% gen_server callbacks 43 | %%%=================================================================== 44 | 45 | init([Port, Auth_method]) -> 46 | State = #state{listen_port = Port, auth_method = Auth_method}, 47 | {ok, Sock} = gen_tcp:listen(State#state.listen_port, [{active, false}, binary, {reuseaddr, true}]), 48 | self() ! accept_connection, 49 | process_flag(trap_exit, true), 50 | {ok, State#state{listen_socket = Sock}}. 51 | 52 | handle_call(_Request, _From, State) -> 53 | Reply = ok, 54 | {reply, Reply, State}. 55 | 56 | handle_cast(_Msg, State) -> 57 | {noreply, State}. 58 | 59 | handle_info(accept_connection, State) -> 60 | case gen_tcp:accept(State#state.listen_socket, 1000) of 61 | {error, timeout} -> 62 | self() ! accept_connection, 63 | {noreply, State}; 64 | {ok, Socket} -> 65 | Pid = proc_lib:spawn_link(fun() -> 66 | socks_server_loop(Socket, State#state.auth_method) 67 | end), 68 | gen_tcp:controlling_process(Socket, Pid), 69 | Pid ! ready, 70 | self() ! accept_connection, 71 | {noreply, State}; 72 | _Err -> 73 | {stop, normal, State} 74 | end; 75 | 76 | handle_info(stop, State) -> 77 | {stop, normal, State}; 78 | 79 | handle_info(_Info, State) -> 80 | {noreply, State}. 81 | 82 | terminate(_Reason, _State) -> 83 | ok. 84 | 85 | code_change(_OldVsn, State, _Extra) -> 86 | {ok, State}. 87 | 88 | %%%=================================================================== 89 | %%% Internal functions 90 | %%%=================================================================== 91 | socks_server_loop(In_socket, Auth_method) -> 92 | receive 93 | ready -> 94 | socks_server_loop(In_socket, Auth_method, <<>>, unauth) 95 | end. 96 | 97 | socks_server_loop(In_socket, Auth_method, Acc, unauth) -> 98 | inet:setopts(In_socket, [{active, once}]), 99 | receive 100 | {tcp, In_socket, Data} -> 101 | Acc_1 = list_to_binary([Acc, Data]), 102 | case Acc_1 of 103 | <<5, ?NO_AUTH>> when Auth_method == ?NO_AUTH -> 104 | ok = gen_tcp:send(In_socket, <<5, ?NO_AUTH>>), 105 | socks_server_loop(In_socket, Auth_method, <<>>, auth_done); 106 | <<5, Num_auth_methods, Auth_methods:Num_auth_methods/binary>> -> 107 | case lists:member(Auth_method, binary_to_list(Auth_methods)) of 108 | true -> 109 | ok = gen_tcp:send(In_socket, <<5, Auth_method>>), 110 | Conn_state = case Auth_method of 111 | ?NO_AUTH -> auth_done; 112 | _ -> auth_pending 113 | end, 114 | socks_server_loop(In_socket, Auth_method, <<>>, Conn_state); 115 | false -> 116 | ok = gen_tcp:send(In_socket, <<5, 16#ff>>), 117 | gen_tcp:close(In_socket) 118 | end; 119 | _ -> 120 | ok = gen_tcp:send(In_socket, <<5, 0>>), 121 | gen_tcp:close(In_socket) 122 | end; 123 | {tcp_closed, In_socket} -> 124 | ok; 125 | {tcp_error, In_socket, _Rsn} -> 126 | ok 127 | end; 128 | socks_server_loop(In_socket, Auth_method, Acc, auth_pending) -> 129 | inet:setopts(In_socket, [{active, once}]), 130 | receive 131 | {tcp, In_socket, Data} -> 132 | Acc_1 = list_to_binary([Acc, Data]), 133 | case Acc_1 of 134 | <<1, U_len, Username:U_len/binary, P_len, Password:P_len/binary>> -> 135 | case check_user_pw(Username, Password) of 136 | ok -> 137 | ok = gen_tcp:send(In_socket, <<1, 0>>), 138 | socks_server_loop(In_socket, Auth_method, <<>>, auth_done); 139 | notok -> 140 | ok = gen_tcp:send(In_socket, <<1, 1>>), 141 | gen_tcp:close(In_socket) 142 | end; 143 | _ -> 144 | socks_server_loop(In_socket, Auth_method, Acc_1, auth_pending) 145 | end; 146 | {tcp_closed, In_socket} -> 147 | ok; 148 | {tcp_error, In_socket, _Rsn} -> 149 | ok 150 | end; 151 | socks_server_loop(In_socket, Auth_method, Acc, auth_done) -> 152 | inet:setopts(In_socket, [{active, once}]), 153 | receive 154 | {tcp, In_socket, Data} -> 155 | Acc_1 = list_to_binary([Acc, Data]), 156 | case Acc_1 of 157 | <<5, 1, 0, Addr_type, Dest_ip:4/binary, Dest_port:16>> when Addr_type == 1-> 158 | handle_connect(In_socket, Addr_type, Dest_ip, Dest_port); 159 | <<5, 1, 0, Addr_type, Dest_len, Dest_hostname:Dest_len/binary, Dest_port:16>> when Addr_type == 3 -> 160 | handle_connect(In_socket, Addr_type, Dest_hostname, Dest_port); 161 | <<5, 1, 0, Addr_type, Dest_ip:16/binary, Dest_port:16>> when Addr_type == 4-> 162 | handle_connect(In_socket, Addr_type, Dest_ip, Dest_port); 163 | _ -> 164 | socks_server_loop(In_socket, Auth_method, Acc_1, auth_done) 165 | end; 166 | {tcp_closed, In_socket} -> 167 | ok; 168 | {tcp_error, In_socket, _Rsn} -> 169 | ok 170 | end. 171 | 172 | handle_connect(In_socket, Addr_type, Dest_host, Dest_port) -> 173 | Dest_host_1 = case Addr_type of 174 | 1 -> 175 | list_to_tuple(binary_to_list(Dest_host)); 176 | 3 -> 177 | binary_to_list(Dest_host); 178 | 4 -> 179 | list_to_tuple(binary_to_list(Dest_host)) 180 | end, 181 | case gen_tcp:connect(Dest_host_1, Dest_port, [binary, {active, once}]) of 182 | {ok, Out_socket} -> 183 | Addr = case Addr_type of 184 | 1 -> 185 | <>; 186 | 3 -> 187 | Len = size(Dest_host), 188 | <>; 189 | 4 -> 190 | <> 191 | end, 192 | ok = gen_tcp:send(In_socket, <<5, 0, 0, Addr_type, Addr/binary>>), 193 | inet:setopts(In_socket, [{active, once}]), 194 | inet:setopts(Out_socket, [{active, once}]), 195 | connected_loop(In_socket, Out_socket); 196 | _Err -> 197 | ok = gen_tcp:send(<<5, 1>>), 198 | gen_tcp:close(In_socket) 199 | end. 200 | 201 | check_user_pw(<<"user">>, <<"password">>) -> 202 | ok; 203 | check_user_pw(_, _) -> 204 | notok. 205 | 206 | connected_loop(In_socket, Out_socket) -> 207 | receive 208 | {tcp, In_socket, Data} -> 209 | inet:setopts(In_socket, [{active, once}]), 210 | ok = gen_tcp:send(Out_socket, Data), 211 | connected_loop(In_socket, Out_socket); 212 | {tcp, Out_socket, Data} -> 213 | inet:setopts(Out_socket, [{active, once}]), 214 | ok = gen_tcp:send(In_socket, Data), 215 | connected_loop(In_socket, Out_socket); 216 | _ -> 217 | ok 218 | end. 219 | -------------------------------------------------------------------------------- /test/ibrowse_test.erl: -------------------------------------------------------------------------------- 1 | %%% File : ibrowse_test.erl 2 | %%% Author : Chandrashekhar Mullaparthi 3 | %%% Description : Test ibrowse 4 | %%% Created : 14 Oct 2003 by Chandrashekhar Mullaparthi 5 | 6 | -module(ibrowse_test). 7 | -export([ 8 | load_test_/3, 9 | send_reqs_1/3, 10 | do_send_req/2, 11 | local_unit_tests/0, 12 | unit_tests/0, 13 | unit_tests/2, 14 | unit_tests_1/3, 15 | verify_chunked_streaming/0, 16 | verify_chunked_streaming/1, 17 | test_chunked_streaming_once/0, 18 | i_do_async_req_list/4, 19 | test_stream_once/3, 20 | test_stream_once/4, 21 | test_20122010/0, 22 | test_20122010/1, 23 | test_pipeline_head_timeout/0, 24 | test_pipeline_head_timeout/1, 25 | do_test_pipeline_head_timeout/4, 26 | test_head_transfer_encoding/0, 27 | test_head_transfer_encoding/1, 28 | test_head_response_with_body/0, 29 | test_head_response_with_body/1, 30 | test_303_response_with_no_body/0, 31 | test_303_response_with_no_body/1, 32 | test_303_response_with_a_body/0, 33 | test_303_response_with_a_body/1, 34 | test_preserve_status_line/0, 35 | test_binary_headers/0, 36 | test_binary_headers/1, 37 | test_dead_lb_pid/0, 38 | test_generate_body_0/0, 39 | test_retry_of_requests/0, 40 | test_retry_of_requests/1, 41 | test_save_to_file_no_content_length/0, 42 | socks5_noauth/0, 43 | socks5_auth_succ/0, 44 | socks5_auth_fail/0 45 | ]). 46 | 47 | -include_lib("ibrowse/include/ibrowse.hrl"). 48 | 49 | %%------------------------------------------------------------------------------ 50 | %% Unit Tests 51 | %%------------------------------------------------------------------------------ 52 | -define(LOCAL_TESTS, [ 53 | {local_test_fun, socks5_noauth, []}, 54 | {local_test_fun, socks5_auth_succ, []}, 55 | {local_test_fun, socks5_auth_fail, []}, 56 | {local_test_fun, test_preserve_status_line, []}, 57 | {local_test_fun, test_save_to_file_no_content_length, []}, 58 | {local_test_fun, test_20122010, []}, 59 | {local_test_fun, test_pipeline_head_timeout, []}, 60 | {local_test_fun, test_head_transfer_encoding, []}, 61 | {local_test_fun, test_head_response_with_body, []}, 62 | {local_test_fun, test_303_response_with_a_body, []}, 63 | {local_test_fun, test_303_response_with_no_body, []}, 64 | {local_test_fun, test_binary_headers, []}, 65 | {local_test_fun, test_dead_lb_pid, []}, 66 | {local_test_fun, test_retry_of_requests, []}, 67 | {local_test_fun, verify_chunked_streaming, []}, 68 | {local_test_fun, test_chunked_streaming_once, []}, 69 | {local_test_fun, test_generate_body_0, []} 70 | ]). 71 | 72 | -define(TEST_LIST, [{"http://intranet/messenger", get}, 73 | {"http://www.google.co.uk", get}, 74 | {"http://www.google.com", get}, 75 | {"http://www.google.com", options}, 76 | {"https://mail.google.com", get}, 77 | {"http://www.sun.com", get}, 78 | {"http://www.oracle.com", get}, 79 | {"http://www.bbc.co.uk", get}, 80 | {"http://www.bbc.co.uk", trace}, 81 | {"http://www.bbc.co.uk", options}, 82 | {"http://yaws.hyber.org", get}, 83 | {"http://jigsaw.w3.org/HTTP/ChunkedScript", get}, 84 | {"http://jigsaw.w3.org/HTTP/TE/foo.txt", get}, 85 | {"http://jigsaw.w3.org/HTTP/TE/bar.txt", get}, 86 | {"http://jigsaw.w3.org/HTTP/connection.html", get}, 87 | {"http://jigsaw.w3.org/HTTP/cc.html", get}, 88 | {"http://jigsaw.w3.org/HTTP/cc-private.html", get}, 89 | {"http://jigsaw.w3.org/HTTP/cc-proxy-revalidate.html", get}, 90 | {"http://jigsaw.w3.org/HTTP/cc-nocache.html", get}, 91 | {"http://jigsaw.w3.org/HTTP/h-content-md5.html", get}, 92 | {"http://jigsaw.w3.org/HTTP/h-retry-after.html", get}, 93 | {"http://jigsaw.w3.org/HTTP/h-retry-after-date.html", get}, 94 | {"http://jigsaw.w3.org/HTTP/neg", get}, 95 | {"http://jigsaw.w3.org/HTTP/negbad", get}, 96 | {"http://jigsaw.w3.org/HTTP/400/toolong/", get}, 97 | {"http://jigsaw.w3.org/HTTP/300/", get}, 98 | {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]}, 99 | {"http://jigsaw.w3.org/HTTP/CL/", get}, 100 | {"http://www.httpwatch.com/httpgallery/chunked/", get}, 101 | {"https://github.com", get, [{ssl_options, [{depth, 2}]}]} 102 | ]). 103 | 104 | socks5_noauth() -> 105 | case ibrowse:send_req("http://localhost:8181/success", [], get, [], 106 | [{socks5_host, "localhost"}, {socks5_port, 8282}], 2000) of 107 | {ok, "200", _, _} -> 108 | success; 109 | Err -> 110 | Err 111 | end. 112 | 113 | socks5_auth_succ() -> 114 | case ibrowse:send_req("http://localhost:8181/success", [], get, [], 115 | [{socks5_host, "localhost"}, {socks5_port, 8383}, 116 | {socks5_user, <<"user">>}, {socks5_password, <<"password">>}], 2000) of 117 | {ok, "200", _, _} -> 118 | success; 119 | Err -> 120 | Err 121 | end. 122 | 123 | socks5_auth_fail() -> 124 | case ibrowse:send_req("http://localhost:8181/success", [], get, [], 125 | [{socks5_host, "localhost"}, {socks5_port, 8282}, 126 | {socks5_user, <<"user">>}, {socks5_password, <<"wrong_password">>}], 2000) of 127 | {error,{conn_failed,{error,unacceptable}}} -> 128 | success; 129 | Err -> 130 | Err 131 | end. 132 | 133 | test_stream_once(Url, Method, Options) -> 134 | test_stream_once(Url, Method, Options, 5000). 135 | 136 | test_stream_once(Url, Method, Options, Timeout) -> 137 | case ibrowse:send_req(Url, [], Method, [], [{stream_to, {self(), once}} | Options], Timeout) of 138 | {ibrowse_req_id, Req_id} -> 139 | case ibrowse:stream_next(Req_id) of 140 | ok -> 141 | test_stream_once(Req_id); 142 | Err -> 143 | Err 144 | end; 145 | Err -> 146 | Err 147 | end. 148 | 149 | test_stream_once(Req_id) -> 150 | receive 151 | {ibrowse_async_headers, Req_id, StatCode, Headers} -> 152 | io:format("Recvd headers~n~p~n", [{ibrowse_async_headers, Req_id, StatCode, Headers}]), 153 | case ibrowse:stream_next(Req_id) of 154 | ok -> 155 | test_stream_once(Req_id); 156 | Err -> 157 | Err 158 | end; 159 | {ibrowse_async_response, Req_id, {error, Err}} -> 160 | io:format("Recvd error: ~p~n", [Err]); 161 | {ibrowse_async_response, Req_id, Body_1} -> 162 | io:format("Recvd body part: ~n~p~n", [{ibrowse_async_response, Req_id, Body_1}]), 163 | case ibrowse:stream_next(Req_id) of 164 | ok -> 165 | test_stream_once(Req_id); 166 | Err -> 167 | Err 168 | end; 169 | {ibrowse_async_response_end, Req_id} -> 170 | ok 171 | end. 172 | 173 | %% Use ibrowse:set_max_sessions/3 and ibrowse:set_max_pipeline_size/3 to 174 | %% tweak settings before running the load test. The defaults are 10 and 10. 175 | load_test_(Url, NumWorkers, NumReqsPerWorker) when is_list(Url), 176 | is_integer(NumWorkers), 177 | is_integer(NumReqsPerWorker), 178 | NumWorkers > 0, 179 | NumReqsPerWorker > 0 -> 180 | proc_lib:spawn(?MODULE, send_reqs_1, [Url, NumWorkers, NumReqsPerWorker]). 181 | 182 | send_reqs_1(Url, NumWorkers, NumReqsPerWorker) -> 183 | Start_time = os:timestamp(), 184 | ets:new(pid_table, [named_table, public]), 185 | ets:new(ibrowse_test_results, [named_table, public]), 186 | ets:new(ibrowse_errors, [named_table, public, ordered_set]), 187 | ets:new(ibrowse_counter, [named_table, public, ordered_set]), 188 | ets:insert(ibrowse_counter, {req_id, 1}), 189 | init_results(), 190 | process_flag(trap_exit, true), 191 | log_msg("Starting spawning of workers...~n", []), 192 | spawn_workers(Url, NumWorkers, NumReqsPerWorker), 193 | log_msg("Finished spawning workers...~n", []), 194 | do_wait(Url), 195 | End_time = os:timestamp(), 196 | log_msg("All workers are done...~n", []), 197 | log_msg("ibrowse_test_results table: ~n~p~n", [ets:tab2list(ibrowse_test_results)]), 198 | log_msg("Start time: ~1000.p~n", [calendar:now_to_local_time(Start_time)]), 199 | log_msg("End time : ~1000.p~n", [calendar:now_to_local_time(End_time)]), 200 | Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000), 201 | log_msg("Elapsed : ~p~n", [Elapsed_time_secs]), 202 | log_msg("Reqs/sec : ~p~n", [round(trunc((NumWorkers*NumReqsPerWorker) / Elapsed_time_secs))]), 203 | dump_errors(). 204 | 205 | init_results() -> 206 | ets:insert(ibrowse_test_results, {crash, 0}), 207 | ets:insert(ibrowse_test_results, {send_failed, 0}), 208 | ets:insert(ibrowse_test_results, {other_error, 0}), 209 | ets:insert(ibrowse_test_results, {success, 0}), 210 | ets:insert(ibrowse_test_results, {retry_later, 0}), 211 | ets:insert(ibrowse_test_results, {trid_mismatch, 0}), 212 | ets:insert(ibrowse_test_results, {success_no_trid, 0}), 213 | ets:insert(ibrowse_test_results, {failed, 0}), 214 | ets:insert(ibrowse_test_results, {timeout, 0}), 215 | ets:insert(ibrowse_test_results, {req_id, 0}). 216 | 217 | spawn_workers(_Url, 0, _) -> 218 | ok; 219 | spawn_workers(Url, NumWorkers, NumReqsPerWorker) -> 220 | Pid = proc_lib:spawn_link(?MODULE, do_send_req, [Url, NumReqsPerWorker]), 221 | ets:insert(pid_table, {Pid, []}), 222 | spawn_workers(Url, NumWorkers - 1, NumReqsPerWorker). 223 | 224 | do_wait(Url) -> 225 | receive 226 | {'EXIT', _, normal} -> 227 | catch ibrowse:show_dest_status(Url), 228 | catch ibrowse:show_dest_status(), 229 | do_wait(Url); 230 | {'EXIT', Pid, Reason} -> 231 | ets:delete(pid_table, Pid), 232 | ets:insert(ibrowse_errors, {Pid, Reason}), 233 | ets:update_counter(ibrowse_test_results, crash, 1), 234 | do_wait(Url); 235 | Msg -> 236 | io:format("Recvd unknown message...~p~n", [Msg]), 237 | do_wait(Url) 238 | after 1000 -> 239 | case ets:info(pid_table, size) of 240 | 0 -> 241 | done; 242 | _ -> 243 | catch ibrowse:show_dest_status(Url), 244 | catch ibrowse:show_dest_status(), 245 | do_wait(Url) 246 | end 247 | end. 248 | 249 | do_send_req(Url, NumReqs) -> 250 | do_send_req_1(Url, NumReqs). 251 | 252 | do_send_req_1(_Url, 0) -> 253 | ets:delete(pid_table, self()); 254 | do_send_req_1(Url, NumReqs) -> 255 | Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)), 256 | case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of 257 | {ok, _Status, Headers, _Body} -> 258 | case lists:keysearch("ib_req_id", 1, Headers) of 259 | {value, {_, Counter}} -> 260 | ets:update_counter(ibrowse_test_results, success, 1); 261 | {value, _} -> 262 | ets:update_counter(ibrowse_test_results, trid_mismatch, 1); 263 | false -> 264 | ets:update_counter(ibrowse_test_results, success_no_trid, 1) 265 | end; 266 | {error, req_timedout} -> 267 | ets:update_counter(ibrowse_test_results, timeout, 1); 268 | {error, send_failed} -> 269 | ets:update_counter(ibrowse_test_results, send_failed, 1); 270 | {error, retry_later} -> 271 | ets:update_counter(ibrowse_test_results, retry_later, 1); 272 | Err -> 273 | ets:insert(ibrowse_errors, {os:timestamp(), Err}), 274 | ets:update_counter(ibrowse_test_results, other_error, 1), 275 | ok 276 | end, 277 | do_send_req_1(Url, NumReqs-1). 278 | 279 | dump_errors() -> 280 | case ets:info(ibrowse_errors, size) of 281 | 0 -> 282 | ok; 283 | _ -> 284 | {A, B, C} = os:timestamp(), 285 | Filename = lists:flatten( 286 | io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])), 287 | case file:open(Filename, [write, delayed_write, raw]) of 288 | {ok, Iod} -> 289 | dump_errors(ets:first(ibrowse_errors), Iod); 290 | Err -> 291 | io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]), 292 | ok 293 | end 294 | end. 295 | 296 | dump_errors('$end_of_table', Iod) -> 297 | file:close(Iod); 298 | dump_errors(Key, Iod) -> 299 | [{_, Term}] = ets:lookup(ibrowse_errors, Key), 300 | file:write(Iod, io_lib:format("~p~n", [Term])), 301 | dump_errors(ets:next(ibrowse_errors, Key), Iod). 302 | 303 | local_unit_tests() -> 304 | unit_tests([], ?LOCAL_TESTS). 305 | 306 | unit_tests() -> 307 | unit_tests([], ?TEST_LIST). 308 | 309 | unit_tests(Options, Test_list) -> 310 | error_logger:tty(false), 311 | application:start(crypto), 312 | application:start(asn1), 313 | application:start(public_key), 314 | application:start(ssl), 315 | (catch ibrowse_test_server:start_server(8181, tcp)), 316 | application:start(ibrowse), 317 | Options_1 = Options ++ [{connect_timeout, 5000}], 318 | Test_timeout = proplists:get_value(test_timeout, Options, 60000), 319 | {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1, Test_list]), 320 | receive 321 | {done, Pid} -> 322 | ok; 323 | {'DOWN', Ref, _, _, Info} -> 324 | io:format("Test process crashed: ~p~n", [Info]) 325 | after Test_timeout -> 326 | exit(Pid, kill), 327 | io:format("Timed out waiting for tests to complete~n", []) 328 | end, 329 | catch ibrowse_test_server:stop_server(8181), 330 | error_logger:tty(true), 331 | ok. 332 | 333 | unit_tests_1(Parent, Options, Test_list) -> 334 | lists:foreach(fun({local_test_fun, Fun_name, Args}) -> 335 | execute_req(local_test_fun, Fun_name, Args); 336 | ({Url, Method}) -> 337 | execute_req(Url, Method, Options); 338 | ({Url, Method, X_Opts}) -> 339 | execute_req(Url, Method, X_Opts ++ Options) 340 | end, Test_list), 341 | Parent ! {done, self()}. 342 | 343 | verify_chunked_streaming() -> 344 | verify_chunked_streaming([]). 345 | 346 | verify_chunked_streaming(Options) -> 347 | io:format("~nVerifying that chunked streaming is working...~n", []), 348 | Url = "http://www.httpwatch.com/httpgallery/chunked/", 349 | io:format(" URL: ~s~n", [Url]), 350 | io:format(" Fetching data without streaming...~n", []), 351 | Result_without_streaming = ibrowse:send_req( 352 | Url, [], get, [], 353 | [{response_format, binary} | Options]), 354 | io:format(" Fetching data with streaming as list...~n", []), 355 | Async_response_list = do_async_req_list( 356 | Url, get, [{response_format, list} | Options]), 357 | io:format(" Fetching data with streaming as binary...~n", []), 358 | Async_response_bin = do_async_req_list( 359 | Url, get, [{response_format, binary} | Options]), 360 | io:format(" Fetching data with streaming as binary, {active, once}...~n", []), 361 | Async_response_bin_once = do_async_req_list( 362 | Url, get, [once, {response_format, binary} | Options]), 363 | Res1 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin), 364 | Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once), 365 | case {Res1, Res2} of 366 | {success, success} -> 367 | io:format(" Chunked streaming working~n", []), 368 | success; 369 | _ -> 370 | ok 371 | end. 372 | 373 | test_chunked_streaming_once() -> 374 | test_chunked_streaming_once([]). 375 | 376 | test_chunked_streaming_once(Options) -> 377 | io:format("~nTesting chunked streaming with the {stream_to, {Pid, once}} option...~n", []), 378 | Url = "http://www.httpwatch.com/httpgallery/chunked/", 379 | io:format(" URL: ~s~n", [Url]), 380 | io:format(" Fetching data with streaming as binary, {active, once}...~n", []), 381 | case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of 382 | {ok, _, _, _} -> 383 | success; 384 | Err -> 385 | io:format(" Fail: ~p~n", [Err]) 386 | end. 387 | 388 | compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) -> 389 | success; 390 | compare_responses({ok, St_code, _, Body_1}, {ok, St_code, _, Body_2}, {ok, St_code, _, Body_3}) -> 391 | case Body_1 of 392 | Body_2 -> 393 | io:format("Body_1 and Body_2 match~n", []); 394 | Body_3 -> 395 | io:format("Body_1 and Body_3 match~n", []); 396 | _ when Body_2 == Body_3 -> 397 | io:format("Body_2 and Body_3 match~n", []); 398 | _ -> 399 | io:format("All three bodies are different!~n", []) 400 | end, 401 | io:format("Body_1 -> ~p~n", [Body_1]), 402 | io:format("Body_2 -> ~p~n", [Body_2]), 403 | io:format("Body_3 -> ~p~n", [Body_3]), 404 | fail_bodies_mismatch; 405 | compare_responses(R1, R2, R3) -> 406 | io:format("R1 -> ~p~n", [R1]), 407 | io:format("R2 -> ~p~n", [R2]), 408 | io:format("R3 -> ~p~n", [R3]), 409 | fail. 410 | 411 | %% do_async_req_list(Url) -> 412 | %% do_async_req_list(Url, get). 413 | 414 | %% do_async_req_list(Url, Method) -> 415 | %% do_async_req_list(Url, Method, [{stream_to, self()}, 416 | %% {stream_chunk_size, 1000}]). 417 | 418 | do_async_req_list(Url, Method, Options) -> 419 | {Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list, 420 | [self(), Url, Method, 421 | Options ++ [{stream_chunk_size, 1000}]]), 422 | %% io:format("Spawned process ~p~n", [Pid]), 423 | wait_for_resp(Pid). 424 | 425 | wait_for_resp(Pid) -> 426 | receive 427 | {async_result, Pid, Res} -> 428 | Res; 429 | {async_result, Other_pid, _} -> 430 | io:format("~p: Waiting for result from ~p: got from ~p~n", [self(), Pid, Other_pid]), 431 | wait_for_resp(Pid); 432 | {'DOWN', _, _, Pid, Reason} -> 433 | {'EXIT', Reason}; 434 | {'DOWN', _, _, _, _} -> 435 | wait_for_resp(Pid); 436 | {'EXIT', _, normal} -> 437 | wait_for_resp(Pid); 438 | Msg -> 439 | io:format("Recvd unknown message: ~p~n", [Msg]), 440 | wait_for_resp(Pid) 441 | after 100000 -> 442 | {error, timeout} 443 | end. 444 | 445 | i_do_async_req_list(Parent, Url, Method, Options) -> 446 | Options_1 = case lists:member(once, Options) of 447 | true -> 448 | [{stream_to, {self(), once}} | (Options -- [once])]; 449 | false -> 450 | [{stream_to, self()} | Options] 451 | end, 452 | Res = ibrowse:send_req(Url, [], Method, [], Options_1), 453 | case Res of 454 | {ibrowse_req_id, Req_id} -> 455 | Result = wait_for_async_resp(Req_id, Options, undefined, undefined, []), 456 | Parent ! {async_result, self(), Result}; 457 | Err -> 458 | Parent ! {async_result, self(), Err} 459 | end. 460 | 461 | wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, Body) -> 462 | receive 463 | {ibrowse_async_headers, Req_id, StatCode, Headers} -> 464 | %% io:format("Recvd headers...~n", []), 465 | maybe_stream_next(Req_id, Options), 466 | wait_for_async_resp(Req_id, Options, StatCode, Headers, Body); 467 | {ibrowse_async_response_end, Req_id} -> 468 | %% io:format("Recvd end of response.~n", []), 469 | Body_1 = list_to_binary(lists:reverse(Body)), 470 | {ok, Acc_Stat_code, Acc_Headers, Body_1}; 471 | {ibrowse_async_response, Req_id, Data} -> 472 | maybe_stream_next(Req_id, Options), 473 | %% io:format("Recvd data...~n", []), 474 | wait_for_async_resp(Req_id, Options, Acc_Stat_code, Acc_Headers, [Data | Body]); 475 | {ibrowse_async_response, Req_id, {error, _} = Err} -> 476 | {ok, Acc_Stat_code, Acc_Headers, Err}; 477 | Err -> 478 | {ok, Acc_Stat_code, Acc_Headers, Err} 479 | after 10000 -> 480 | {timeout, Acc_Stat_code, Acc_Headers, Body} 481 | end. 482 | 483 | maybe_stream_next(Req_id, Options) -> 484 | case lists:member(once, Options) of 485 | true -> 486 | ibrowse:stream_next(Req_id); 487 | false -> 488 | ok 489 | end. 490 | 491 | execute_req(local_test_fun, Method, Args) -> 492 | reset_ibrowse(), 493 | Result = (catch apply(?MODULE, Method, Args)), 494 | io:format(" ~-54.54w: ", [Method]), 495 | io:format("~p~n", [Result]); 496 | execute_req(Url, Method, Options) -> 497 | io:format("~7.7w, ~50.50s: ", [Method, Url]), 498 | Result = (catch ibrowse:send_req(Url, [], Method, [], Options)), 499 | case Result of 500 | {ok, SCode, _H, _B} -> 501 | io:format("Status code: ~p~n", [SCode]); 502 | Err -> 503 | io:format("~p~n", [Err]) 504 | end. 505 | 506 | log_msg(Fmt, Args) -> 507 | io:format("~s -- " ++ Fmt, 508 | [ibrowse_lib:printable_date() | Args]). 509 | 510 | %%------------------------------------------------------------------------------ 511 | %% Test what happens when the response to a HEAD request is a 512 | %% Chunked-Encoding response with a non-empty body. Issue #67 on 513 | %% Github 514 | %% ------------------------------------------------------------------------------ 515 | test_head_transfer_encoding() -> 516 | clear_msg_q(), 517 | test_head_transfer_encoding("http://localhost:8181/ibrowse_head_test"). 518 | 519 | test_head_transfer_encoding(Url) -> 520 | case ibrowse:send_req(Url, [], head) of 521 | {ok, "200", _, _} -> 522 | success; 523 | Res -> 524 | {test_failed, Res} 525 | end. 526 | 527 | %%------------------------------------------------------------------------------ 528 | %% Test what happens when the response to a HEAD request is a 529 | %% Chunked-Encoding response with a non-empty body. Issue #67 on 530 | %% Github 531 | %% ------------------------------------------------------------------------------ 532 | test_binary_headers() -> 533 | clear_msg_q(), 534 | test_binary_headers("http://localhost:8181/ibrowse_echo_header"). 535 | 536 | test_binary_headers(Url) -> 537 | case ibrowse:send_req(Url, [{<<"x-binary">>, <<"x-header">>}], get) of 538 | {ok, "200", Headers, _} -> 539 | case proplists:get_value("x-binary", Headers) of 540 | "x-header" -> 541 | success; 542 | V -> 543 | {fail, V} 544 | end; 545 | Res -> 546 | {test_failed, Res} 547 | end. 548 | 549 | %%------------------------------------------------------------------------------ 550 | %% Test what happens when the response to a HEAD request is a 551 | %% Chunked-Encoding response with a non-empty body. Issue #67 on 552 | %% Github 553 | %% ------------------------------------------------------------------------------ 554 | test_head_response_with_body() -> 555 | clear_msg_q(), 556 | test_head_response_with_body("http://localhost:8181/ibrowse_head_transfer_enc"). 557 | 558 | test_head_response_with_body(Url) -> 559 | case ibrowse:send_req(Url, [], head, [], [{workaround, head_response_with_body}]) of 560 | {ok, "400", _, _} -> 561 | success; 562 | Res -> 563 | {test_failed, Res} 564 | end. 565 | 566 | %%------------------------------------------------------------------------------ 567 | %% Test what happens when a 303 response has no body 568 | %% Github issue #97 569 | %% ------------------------------------------------------------------------------ 570 | test_303_response_with_no_body() -> 571 | clear_msg_q(), 572 | test_303_response_with_no_body("http://localhost:8181/ibrowse_303_no_body_test"). 573 | 574 | test_303_response_with_no_body(Url) -> 575 | ibrowse:add_config([{allow_303_with_no_body, true}]), 576 | case ibrowse:send_req(Url, [], post) of 577 | {ok, "303", _, _} -> 578 | success; 579 | Res -> 580 | {test_failed, Res} 581 | end. 582 | 583 | %% Make sure we don't break requests that do have a body. 584 | test_303_response_with_a_body() -> 585 | clear_msg_q(), 586 | test_303_response_with_no_body("http://localhost:8181/ibrowse_303_with_body_test"). 587 | 588 | test_303_response_with_a_body(Url) -> 589 | ibrowse:add_config([{allow_303_with_no_body, true}]), 590 | case ibrowse:send_req(Url, [], post) of 591 | {ok, "303", _, "abcde"} -> 592 | success; 593 | Res -> 594 | {test_failed, Res} 595 | end. 596 | 597 | %% Test that the 'preserve_status_line' option works as expected 598 | test_preserve_status_line() -> 599 | case ibrowse:send_req("http://localhost:8181/ibrowse_preserve_status_line", [], get, [], 600 | [{preserve_status_line, true}]) of 601 | {ok, "200", [{ibrowse_status_line,<<"HTTP/1.1 200 OKBlah">>} | _], _} -> 602 | success; 603 | Res -> 604 | {test_failed, Res} 605 | end. 606 | 607 | %%------------------------------------------------------------------------------ 608 | %% Test that when the save_response_to_file option is used with a server which 609 | %% does not send the Content-Length header, the response is saved correctly to 610 | %% a file 611 | %%------------------------------------------------------------------------------ 612 | test_save_to_file_no_content_length() -> 613 | clear_msg_q(), 614 | {{Y, M, D}, {H, Mi, S}} = calendar:local_time(), 615 | Test_file = filename:join 616 | ([".", 617 | lists:flatten( 618 | io_lib:format("test_save_to_file_no_content_length_~p~p~p_~p~p~p.txt", [Y, M, D, H, Mi, S]))]), 619 | try 620 | case ibrowse:send_req("http://localhost:8181/ibrowse_send_file_conn_close", [], get, [], 621 | [{save_response_to_file, Test_file}]) of 622 | {ok, "200", _, {file, Test_file}} -> 623 | success; 624 | Res -> 625 | {test_failed, Res} 626 | end 627 | after 628 | file:delete(Test_file) 629 | end. 630 | 631 | %%------------------------------------------------------------------------------ 632 | %% Test that retry of requests happens correctly, and that ibrowse doesn't retry 633 | %% if there is not enough time left 634 | %%------------------------------------------------------------------------------ 635 | test_retry_of_requests() -> 636 | clear_msg_q(), 637 | test_retry_of_requests("http://localhost:8181/ibrowse_handle_one_request_only_with_delay"). 638 | 639 | test_retry_of_requests(Url) -> 640 | reset_ibrowse(), 641 | Timeout_1 = 2050, 642 | Res_1 = test_retry_of_requests(Url, Timeout_1), 643 | case lists:filter(fun({_Pid, {ok, "200", _, _}}) -> 644 | true; 645 | (_) -> false 646 | end, Res_1) of 647 | [_|_] = X -> 648 | Res_1_1 = Res_1 -- X, 649 | case lists:all( 650 | fun({_Pid, {error, retry_later}}) -> 651 | true; 652 | (_) -> 653 | false 654 | end, Res_1_1) of 655 | true -> 656 | ok; 657 | false -> 658 | exit({failed, Timeout_1, Res_1}) 659 | end; 660 | _ -> 661 | exit({failed, Timeout_1, Res_1}) 662 | end, 663 | Timeout_2 = 2200, 664 | Res_2 = test_retry_of_requests(Url, Timeout_2), 665 | case lists:filter(fun({_Pid, {ok, "200", _, _}}) -> 666 | true; 667 | (_) -> false 668 | end, Res_2) of 669 | [_|_] = Res_2_X -> 670 | Res_2_1 = Res_2 -- Res_2_X, 671 | case lists:all( 672 | fun({_Pid, {error, X_err_2}}) -> 673 | (X_err_2 == retry_later) orelse (X_err_2 == req_timedout); 674 | (_) -> 675 | false 676 | end, Res_2_1) of 677 | true -> 678 | ok; 679 | false -> 680 | exit({failed, {?MODULE, ?LINE}, Timeout_2, Res_2}) 681 | end; 682 | _ -> 683 | exit({failed, {?MODULE, ?LINE}, Timeout_2, Res_2}) 684 | end, 685 | success. 686 | 687 | test_retry_of_requests(Url, Timeout) -> 688 | #url{host = Host, port = Port} = ibrowse_lib:parse_url(Url), 689 | ibrowse:set_max_sessions(Host, Port, 1), 690 | Parent = self(), 691 | Pids = lists:map(fun(_) -> 692 | spawn(fun() -> 693 | Res = (catch ibrowse:send_req(Url, [], get, [], [], Timeout)), 694 | Parent ! {self(), Res} 695 | end) 696 | end, lists:seq(1,10)), 697 | accumulate_worker_resp(Pids). 698 | 699 | %%------------------------------------------------------------------------------ 700 | %% Test what happens when the request at the head of a pipeline times out 701 | %%------------------------------------------------------------------------------ 702 | test_pipeline_head_timeout() -> 703 | clear_msg_q(), 704 | test_pipeline_head_timeout("http://localhost:8181/ibrowse_inac_timeout_test"). 705 | 706 | test_pipeline_head_timeout(Url) -> 707 | {ok, Pid} = ibrowse:spawn_worker_process(Url), 708 | Fixed_timeout = 2000, 709 | Test_parent = self(), 710 | Fun = fun({fixed, Timeout}) -> 711 | X_pid = spawn(fun() -> 712 | do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout) 713 | end), 714 | %% io:format("Pid ~p with a fixed timeout~n", [X_pid]), 715 | X_pid; 716 | (Timeout_mult) -> 717 | Timeout = Fixed_timeout + Timeout_mult*1000, 718 | X_pid = spawn(fun() -> 719 | do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout) 720 | end), 721 | %% io:format("Pid ~p with a timeout of ~p~n", [X_pid, Timeout]), 722 | X_pid 723 | end, 724 | Pids = [Fun(X) || X <- [{fixed, Fixed_timeout} | lists:seq(1,10)]], 725 | Result = accumulate_worker_resp(Pids), 726 | case lists:all(fun({_, X_res}) -> 727 | (X_res == {error,req_timedout}) orelse (X_res == {error, connection_closed}) 728 | end, Result) of 729 | true -> 730 | success; 731 | false -> 732 | {test_failed, Result} 733 | end. 734 | 735 | do_test_pipeline_head_timeout(Url, Pid, Test_parent, Req_timeout) -> 736 | Resp = ibrowse:send_req_direct( 737 | Pid, 738 | Url, 739 | [], get, [], 740 | [{socket_options,[{keepalive,true}]}, 741 | {inactivity_timeout,180000}, 742 | {connect_timeout,180000}], Req_timeout), 743 | Test_parent ! {self(), Resp}. 744 | 745 | accumulate_worker_resp(Pids) -> 746 | accumulate_worker_resp(Pids, []). 747 | 748 | accumulate_worker_resp([_ | _] = Pids, Acc) -> 749 | receive 750 | {Pid, Res} when is_pid(Pid) -> 751 | accumulate_worker_resp(Pids -- [Pid], [{Pid, Res} | Acc]); 752 | Err -> 753 | io:format("Received unexpected: ~p~n", [Err]) 754 | end; 755 | accumulate_worker_resp([], Acc) -> 756 | lists:reverse(Acc). 757 | 758 | clear_msg_q() -> 759 | receive 760 | _ -> 761 | clear_msg_q() 762 | after 0 -> 763 | ok 764 | end. 765 | %%------------------------------------------------------------------------------ 766 | %% 767 | %%------------------------------------------------------------------------------ 768 | 769 | test_20122010() -> 770 | test_20122010("http://localhost:8181"). 771 | 772 | test_20122010(Url) -> 773 | {ok, Pid} = ibrowse:spawn_worker_process(Url), 774 | Expected_resp = <<"1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18-19-20-21-22-23-24-25-26-27-28-29-30-31-32-33-34-35-36-37-38-39-40-41-42-43-44-45-46-47-48-49-50-51-52-53-54-55-56-57-58-59-60-61-62-63-64-65-66-67-68-69-70-71-72-73-74-75-76-77-78-79-80-81-82-83-84-85-86-87-88-89-90-91-92-93-94-95-96-97-98-99-100">>, 775 | Test_parent = self(), 776 | Fun = fun() -> 777 | do_test_20122010(Url, Pid, Expected_resp, Test_parent) 778 | end, 779 | Pids = [erlang:spawn_monitor(Fun) || _ <- lists:seq(1,10)], 780 | wait_for_workers(Pids). 781 | 782 | wait_for_workers([{Pid, _Ref} | Pids]) -> 783 | receive 784 | {Pid, success} -> 785 | wait_for_workers(Pids) 786 | after 60000 -> 787 | test_failed 788 | end; 789 | wait_for_workers([]) -> 790 | success. 791 | 792 | do_test_20122010(Url, Pid, Expected_resp, Test_parent) -> 793 | do_test_20122010(10, Url, Pid, Expected_resp, Test_parent). 794 | 795 | do_test_20122010(0, _Url, _Pid, _Expected_resp, Test_parent) -> 796 | Test_parent ! {self(), success}; 797 | do_test_20122010(Rem_count, Url, Pid, Expected_resp, Test_parent) -> 798 | {ibrowse_req_id, Req_id} = ibrowse:send_req_direct( 799 | Pid, 800 | Url ++ "/ibrowse_stream_once_chunk_pipeline_test", 801 | [], get, [], 802 | [{stream_to, {self(), once}}, 803 | {inactivity_timeout, 10000}, 804 | {include_ibrowse_req_id, true}]), 805 | do_trace("~p -- sent request ~1000.p~n", [self(), Req_id]), 806 | Req_id_str = lists:flatten(io_lib:format("~1000.p",[Req_id])), 807 | receive 808 | {ibrowse_async_headers, Req_id, "200", Headers} -> 809 | case lists:keysearch("x-ibrowse-request-id", 1, Headers) of 810 | {value, {_, Req_id_str}} -> 811 | ok; 812 | {value, {_, Req_id_1}} -> 813 | do_trace("~p -- Sent req-id: ~1000.p. Recvd: ~1000.p~n", 814 | [self(), Req_id, Req_id_1]), 815 | exit(req_id_mismatch) 816 | end 817 | after 5000 -> 818 | do_trace("~p -- response headers not received~n", [self()]), 819 | exit({timeout, test_failed}) 820 | end, 821 | do_trace("~p -- response headers received~n", [self()]), 822 | ok = ibrowse:stream_next(Req_id), 823 | case do_test_20122010_1(Expected_resp, Req_id, []) of 824 | true -> 825 | do_test_20122010(Rem_count - 1, Url, Pid, Expected_resp, Test_parent); 826 | false -> 827 | Test_parent ! {self(), failed} 828 | end. 829 | 830 | do_test_20122010_1(Expected_resp, Req_id, Acc) -> 831 | receive 832 | {ibrowse_async_response, Req_id, Body_part} -> 833 | ok = ibrowse:stream_next(Req_id), 834 | do_test_20122010_1(Expected_resp, Req_id, [Body_part | Acc]); 835 | {ibrowse_async_response_end, Req_id} -> 836 | Acc_1 = list_to_binary(lists:reverse(Acc)), 837 | Result = Acc_1 == Expected_resp, 838 | do_trace("~p -- End of response. Result: ~p~n", [self(), Result]), 839 | Result 840 | after 1000 -> 841 | exit({timeout, test_failed}) 842 | end. 843 | 844 | %%------------------------------------------------------------------------------ 845 | %% Test requests where body is generated using a Fun 846 | %%------------------------------------------------------------------------------ 847 | test_generate_body_0() -> 848 | Tid = ets:new(ibrowse_test_state, [public]), 849 | try 850 | Body_1 = <<"Part 1 of the body">>, 851 | Body_2 = <<"Part 2 of the body\r\n">>, 852 | Size = size(Body_1) + size(Body_2), 853 | Body = list_to_binary([Body_1, Body_2]), 854 | Fun = fun() -> 855 | case ets:lookup(Tid, body_gen_state) of 856 | [] -> 857 | ets:insert(Tid, {body_gen_state, 1}), 858 | {ok, Body_1}; 859 | [{_, 1}]-> 860 | ets:insert(Tid, {body_gen_state, 2}), 861 | {ok, Body_2}; 862 | [{_, 2}] -> 863 | eof 864 | end 865 | end, 866 | case ibrowse:send_req("http://localhost:8181/echo_body", 867 | [{"Content-Length", Size}], 868 | post, 869 | Fun, 870 | [{response_format, binary}, 871 | {http_vsn, {1,1}}]) of 872 | {ok, "200", _, Body} -> 873 | success; 874 | Err -> 875 | io:format("Test failed : ~p~n", [Err]), 876 | {test_failed, Err} 877 | end 878 | after 879 | ets:delete(Tid) 880 | end. 881 | 882 | %%------------------------------------------------------------------------------ 883 | %% Test that when an lb process dies, its entry is removed from the ibrowse_lb 884 | %% table by the next requestor and replaced with a new process 885 | %%------------------------------------------------------------------------------ 886 | test_dead_lb_pid() -> 887 | {Host, Port} = {"localhost", 8181}, 888 | Url = "http://" ++ Host ++ ":" ++ integer_to_list(Port), 889 | {ok, "200", _, _} = ibrowse:send_req(Url, [], get), 890 | [{lb_pid, {Host, Port}, Pid, _}] = ets:lookup(ibrowse_lb, {Host, Port}), 891 | true = exit(Pid, kill), 892 | false = is_process_alive(Pid), 893 | {ok, "200", _, _} = ibrowse:send_req(Url, [], get), 894 | [{lb_pid, {Host, Port}, NewPid, _}] = ets:lookup(ibrowse_lb, {Host, Port}), 895 | true = NewPid /= Pid, 896 | true = is_process_alive(NewPid), 897 | success. 898 | 899 | do_trace(Fmt, Args) -> 900 | do_trace(get(my_trace_flag), Fmt, Args). 901 | 902 | do_trace(true, Fmt, Args) -> 903 | io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]); 904 | do_trace(_, _, _) -> 905 | ok. 906 | 907 | reset_ibrowse() -> 908 | application:stop(ibrowse), 909 | application:start(ibrowse). 910 | -------------------------------------------------------------------------------- /test/ibrowse_test_server.erl: -------------------------------------------------------------------------------- 1 | %%% File : ibrowse_test_server.erl 2 | %%% Author : Chandrashekhar Mullaparthi 3 | %%% Description : A server to simulate various test scenarios 4 | %%% Created : 17 Oct 2010 by Chandrashekhar Mullaparthi 5 | 6 | -module(ibrowse_test_server). 7 | -export([ 8 | start_server/0, 9 | start_server/2, 10 | stop_server/1, 11 | get_conn_pipeline_depth/0 12 | ]). 13 | 14 | -ifdef(new_rand). 15 | -define(RAND, rand). 16 | -else. 17 | -define(RAND, random). 18 | -endif. 19 | 20 | -record(request, {method, uri, version, headers = [], body = [], state}). 21 | 22 | -define(dec2hex(X), erlang:integer_to_list(X, 16)). 23 | -define(ACCEPT_TIMEOUT_MS, 10000). 24 | -define(CONN_PIPELINE_DEPTH, conn_pipeline_depth). 25 | 26 | start_server() -> 27 | start_server(8181, tcp). 28 | 29 | start_server(Port, Sock_type) -> 30 | Fun = fun() -> 31 | Proc_name = server_proc_name(Port), 32 | case whereis(Proc_name) of 33 | undefined -> 34 | register(Proc_name, self()), 35 | ets:new(?CONN_PIPELINE_DEPTH, [named_table, public, set]), 36 | case do_listen(Sock_type, Port, [{active, false}, 37 | {reuseaddr, true}, 38 | {nodelay, true}, 39 | {packet, http}]) of 40 | {ok, Sock} -> 41 | do_trace("Server listening on port: ~p~n", [Port]), 42 | accept_loop(Sock, Sock_type); 43 | Err -> 44 | erlang:error( 45 | lists:flatten( 46 | io_lib:format( 47 | "Failed to start server on port ~p. ~p~n", 48 | [Port, Err]))), 49 | exit({listen_error, Err}) 50 | end; 51 | _X -> 52 | ok 53 | end 54 | end, 55 | spawn_link(Fun), 56 | ibrowse_socks_server:start(8282, 0), %% No auth 57 | ibrowse_socks_server:start(8383, 2). %% Username/Password auth 58 | 59 | stop_server(Port) -> 60 | catch server_proc_name(Port) ! stop, 61 | ibrowse_socks_server:stop(8282), 62 | ibrowse_socks_server:stop(8383), 63 | timer:sleep(2000), % wait for server to receive msg and unregister 64 | ok. 65 | 66 | get_conn_pipeline_depth() -> 67 | ets:tab2list(?CONN_PIPELINE_DEPTH). 68 | 69 | server_proc_name(Port) -> 70 | list_to_atom("ibrowse_test_server_"++integer_to_list(Port)). 71 | 72 | do_listen(tcp, Port, Opts) -> 73 | gen_tcp:listen(Port, Opts); 74 | do_listen(ssl, Port, Opts) -> 75 | application:start(crypto), 76 | application:start(ssl), 77 | ssl:listen(Port, Opts). 78 | 79 | -ifdef(OTP_RELEASE). 80 | 81 | do_accept(tcp, Listen_sock) -> 82 | gen_tcp:accept(Listen_sock, ?ACCEPT_TIMEOUT_MS); 83 | do_accept(ssl, Listen_sock) -> 84 | ssl:handshake(Listen_sock, ?ACCEPT_TIMEOUT_MS). 85 | 86 | -else. 87 | 88 | do_accept(tcp, Listen_sock) -> 89 | gen_tcp:accept(Listen_sock, ?ACCEPT_TIMEOUT_MS); 90 | do_accept(ssl, Listen_sock) -> 91 | ssl:ssl_accept(Listen_sock, ?ACCEPT_TIMEOUT_MS). 92 | 93 | -endif. 94 | 95 | accept_loop(Sock, Sock_type) -> 96 | case do_accept(Sock_type, Sock) of 97 | {ok, Conn} -> 98 | Pid = spawn_link(fun() -> connection(Conn, Sock_type) end), 99 | set_controlling_process(Conn, Sock_type, Pid), 100 | accept_loop(Sock, Sock_type); 101 | {error, timeout} -> 102 | receive 103 | stop -> 104 | ok 105 | after 10 -> 106 | accept_loop(Sock, Sock_type) 107 | end; 108 | Err -> 109 | Err 110 | end. 111 | 112 | connection(Conn, Sock_type) -> 113 | catch ets:insert(?CONN_PIPELINE_DEPTH, {self(), 0}), 114 | try 115 | inet:setopts(Conn, [{packet, http}, {active, true}]), 116 | server_loop(Conn, Sock_type, #request{}) 117 | after 118 | catch ets:delete(?CONN_PIPELINE_DEPTH, self()) 119 | end. 120 | 121 | set_controlling_process(Sock, tcp, Pid) -> 122 | gen_tcp:controlling_process(Sock, Pid); 123 | set_controlling_process(Sock, ssl, Pid) -> 124 | ssl:controlling_process(Sock, Pid). 125 | 126 | setopts(Sock, tcp, Opts) -> 127 | inet:setopts(Sock, Opts); 128 | setopts(Sock, ssl, Opts) -> 129 | ssl:setopts(Sock, Opts). 130 | 131 | server_loop(Sock, Sock_type, #request{headers = Headers} = Req) -> 132 | receive 133 | {http, Sock, {http_request, HttpMethod, HttpUri, HttpVersion}} -> 134 | catch ets:update_counter(?CONN_PIPELINE_DEPTH, self(), 1), 135 | server_loop(Sock, Sock_type, Req#request{method = HttpMethod, 136 | uri = HttpUri, 137 | version = HttpVersion}); 138 | {http, Sock, {http_header, _, _, _, _} = H} -> 139 | server_loop(Sock, Sock_type, Req#request{headers = [H | Headers]}); 140 | {http, Sock, http_eoh} -> 141 | case process_request(Sock, Sock_type, Req) of 142 | close_connection -> 143 | gen_tcp:shutdown(Sock, read_write); 144 | not_done -> 145 | ok; 146 | collect_body -> 147 | server_loop(Sock, Sock_type, Req#request{state = collect_body}); 148 | _ -> 149 | catch ets:update_counter(?CONN_PIPELINE_DEPTH, self(), -1) 150 | end, 151 | server_loop(Sock, Sock_type, #request{}); 152 | {http, Sock, {http_error, Packet}} when Req#request.state == collect_body -> 153 | Req_1 = Req#request{body = list_to_binary([Packet, Req#request.body])}, 154 | case process_request(Sock, Sock_type, Req_1) of 155 | close_connection -> 156 | gen_tcp:shutdown(Sock, read_write); 157 | ok -> 158 | server_loop(Sock, Sock_type, #request{}); 159 | collect_body -> 160 | server_loop(Sock, Sock_type, Req_1) 161 | end; 162 | {http, Sock, {http_error, Err}} -> 163 | io:format("Error parsing HTTP request:~n" 164 | "Req so far : ~p~n" 165 | "Err : ~p", [Req, Err]), 166 | exit({http_error, Err}); 167 | {setopts, Opts} -> 168 | setopts(Sock, Sock_type, Opts), 169 | server_loop(Sock, Sock_type, Req); 170 | {tcp_closed, Sock} -> 171 | do_trace("Client closed connection~n", []), 172 | ok; 173 | Other -> 174 | io:format("Recvd unknown msg: ~p~n", [Other]), 175 | exit({unknown_msg, Other}) 176 | after 120000 -> 177 | do_trace("Timing out client connection~n", []), 178 | ok 179 | end. 180 | 181 | do_trace(Fmt, Args) -> 182 | do_trace(get(my_trace_flag), Fmt, Args). 183 | 184 | do_trace(true, Fmt, Args) -> 185 | io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]); 186 | do_trace(_, _, _) -> 187 | ok. 188 | 189 | process_request(Sock, Sock_type, 190 | #request{method='GET', 191 | headers = Headers, 192 | uri = {abs_path, "/ibrowse_stream_once_chunk_pipeline_test"}} = Req) -> 193 | Req_id = case lists:keysearch("X-Ibrowse-Request-Id", 3, Headers) of 194 | false -> 195 | ""; 196 | {value, {http_header, _, _, _, Req_id_1}} -> 197 | Req_id_1 198 | end, 199 | Req_id_header = ["x-ibrowse-request-id: ", Req_id, "\r\n"], 200 | do_trace("Recvd req: ~p~n", [Req]), 201 | Body = string:join([integer_to_list(X) || X <- lists:seq(1,100)], "-"), 202 | Chunked_body = chunk_request_body(Body, 50), 203 | Resp_1 = [<<"HTTP/1.1 200 OK\r\n">>, 204 | Req_id_header, 205 | <<"Transfer-Encoding: chunked\r\n\r\n">>], 206 | Resp_2 = Chunked_body, 207 | do_send(Sock, Sock_type, Resp_1), 208 | timer:sleep(100), 209 | do_send(Sock, Sock_type, Resp_2); 210 | process_request(Sock, Sock_type, 211 | #request{method='GET', 212 | headers = _Headers, 213 | uri = {abs_path, "/ibrowse_inac_timeout_test"}} = Req) -> 214 | do_trace("Recvd req: ~p. Sleeping for 30 secs...~n", [Req]), 215 | timer:sleep(3000), 216 | do_trace("...Sending response now.~n", []), 217 | Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>, 218 | do_send(Sock, Sock_type, Resp); 219 | process_request(Sock, Sock_type, 220 | #request{method='HEAD', 221 | headers = _Headers, 222 | uri = {abs_path, "/ibrowse_head_transfer_enc"}}) -> 223 | Resp = <<"HTTP/1.1 400 Bad Request\r\nServer: Apache-Coyote/1.1\r\nContent-Length:5\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\n\r\nabcde">>, 224 | do_send(Sock, Sock_type, Resp); 225 | process_request(Sock, Sock_type, 226 | #request{method='POST', 227 | headers = Headers, 228 | uri = {abs_path, "/echo_body"}, 229 | body = Body}) -> 230 | Content_len = get_content_length(Headers), 231 | case iolist_size(Body) == Content_len of 232 | true -> 233 | Resp = [<<"HTTP/1.1 200 OK\r\nContent-Length: ">>, integer_to_list(Content_len), <<"\r\nServer: ibrowse_test_server\r\n\r\n">>, Body], 234 | do_send(Sock, Sock_type, list_to_binary(Resp)); 235 | false -> 236 | collect_body 237 | end; 238 | process_request(Sock, Sock_type, 239 | #request{method='GET', 240 | headers = Headers, 241 | uri = {abs_path, "/ibrowse_echo_header"}}) -> 242 | Tag = "x-binary", 243 | Headers_1 = [{to_lower(X), to_lower(Y)} || {http_header, _, X, _, Y} <- Headers], 244 | X_binary_header_val = case lists:keysearch(Tag, 1, Headers_1) of 245 | false -> 246 | "not_found"; 247 | {value, {_, V}} -> 248 | V 249 | end, 250 | Resp = [<<"HTTP/1.1 200 OK\r\n">>, 251 | <<"Server: ibrowse_test\r\n">>, 252 | Tag, ": ", X_binary_header_val, "\r\n", 253 | <<"Content-Length: 0\r\n\r\n">>], 254 | do_send(Sock, Sock_type, Resp); 255 | 256 | process_request(Sock, Sock_type, 257 | #request{method='HEAD', 258 | headers = _Headers, 259 | uri = {abs_path, "/ibrowse_head_test"}}) -> 260 | Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\Date: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>, 261 | do_send(Sock, Sock_type, Resp); 262 | process_request(Sock, Sock_type, 263 | #request{method='POST', 264 | headers = _Headers, 265 | uri = {abs_path, "/ibrowse_303_no_body_test"}}) -> 266 | Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\n\r\n">>, 267 | do_send(Sock, Sock_type, Resp); 268 | process_request(Sock, Sock_type, 269 | #request{method='POST', 270 | headers = _Headers, 271 | uri = {abs_path, "/ibrowse_303_with_body_test"}}) -> 272 | Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\nContent-Length: 5\r\n\r\nabcde">>, 273 | do_send(Sock, Sock_type, Resp); 274 | process_request(Sock, Sock_type, 275 | #request{method='GET', 276 | headers = _Headers, 277 | uri = {abs_path, "/ibrowse_preserve_status_line"}}) -> 278 | Resp = <<"HTTP/1.1 200 OKBlah\r\nContent-Length: 5\r\n\r\nabcde">>, 279 | do_send(Sock, Sock_type, Resp); 280 | process_request(Sock, Sock_type, 281 | #request{method='GET', 282 | headers = _Headers, 283 | uri = {abs_path, "/ibrowse_handle_one_request_only_with_delay"}}) -> 284 | timer:sleep(2000), 285 | Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>, 286 | do_send(Sock, Sock_type, Resp), 287 | close_connection; 288 | process_request(Sock, Sock_type, 289 | #request{method='GET', 290 | headers = _Headers, 291 | uri = {abs_path, "/ibrowse_handle_one_request_only"}}) -> 292 | Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>, 293 | do_send(Sock, Sock_type, Resp), 294 | close_connection; 295 | process_request(Sock, Sock_type, 296 | #request{method='GET', 297 | headers = _Headers, 298 | uri = {abs_path, "/ibrowse_send_file_conn_close"}}) -> 299 | Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\nblahblah-">>, 300 | do_send(Sock, Sock_type, Resp), 301 | timer:sleep(1000), 302 | do_send(Sock, Sock_type, <<"blahblah">>), 303 | close_connection; 304 | process_request(_Sock, _Sock_type, #request{uri = {abs_path, "/never_respond"} } ) -> 305 | not_done; 306 | process_request(Sock, Sock_type, Req) -> 307 | do_trace("Recvd req: ~p~n", [Req]), 308 | Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>, 309 | do_send(Sock, Sock_type, Resp), 310 | timer:sleep(?RAND:uniform(100)). 311 | 312 | do_send(Sock, tcp, Resp) -> 313 | gen_tcp:send(Sock, Resp); 314 | do_send(Sock, ssl, Resp) -> 315 | ssl:send(Sock, Resp). 316 | 317 | %%------------------------------------------------------------------------------ 318 | %% Utility functions 319 | %%------------------------------------------------------------------------------ 320 | 321 | chunk_request_body(Body, _ChunkSize) when is_tuple(Body) orelse 322 | is_function(Body) -> 323 | Body; 324 | chunk_request_body(Body, ChunkSize) -> 325 | chunk_request_body(Body, ChunkSize, []). 326 | 327 | chunk_request_body(Body, _ChunkSize, Acc) when Body == <<>>; Body == [] -> 328 | LastChunk = "0\r\n", 329 | lists:reverse(["\r\n", LastChunk | Acc]); 330 | chunk_request_body(Body, ChunkSize, Acc) when is_binary(Body), 331 | size(Body) >= ChunkSize -> 332 | <> = Body, 333 | Chunk = [?dec2hex(ChunkSize),"\r\n", 334 | ChunkBody, "\r\n"], 335 | chunk_request_body(Rest, ChunkSize, [Chunk | Acc]); 336 | chunk_request_body(Body, _ChunkSize, Acc) when is_binary(Body) -> 337 | BodySize = size(Body), 338 | Chunk = [?dec2hex(BodySize),"\r\n", 339 | Body, "\r\n"], 340 | LastChunk = "0\r\n", 341 | lists:reverse(["\r\n", LastChunk, Chunk | Acc]); 342 | chunk_request_body(Body, ChunkSize, Acc) when length(Body) >= ChunkSize -> 343 | {ChunkBody, Rest} = split_list_at(Body, ChunkSize), 344 | Chunk = [?dec2hex(ChunkSize),"\r\n", 345 | ChunkBody, "\r\n"], 346 | chunk_request_body(Rest, ChunkSize, [Chunk | Acc]); 347 | chunk_request_body(Body, _ChunkSize, Acc) when is_list(Body) -> 348 | BodySize = length(Body), 349 | Chunk = [?dec2hex(BodySize),"\r\n", 350 | Body, "\r\n"], 351 | LastChunk = "0\r\n", 352 | lists:reverse(["\r\n", LastChunk, Chunk | Acc]). 353 | 354 | split_list_at(List, N) -> 355 | split_list_at(List, N, []). 356 | 357 | split_list_at([], _, Acc) -> 358 | {lists:reverse(Acc), []}; 359 | split_list_at(List2, 0, List1) -> 360 | {lists:reverse(List1), List2}; 361 | split_list_at([H | List2], N, List1) -> 362 | split_list_at(List2, N-1, [H | List1]). 363 | 364 | to_lower(X) when is_atom(X) -> 365 | list_to_atom(to_lower(atom_to_list(X))); 366 | to_lower(X) when is_list(X) -> 367 | string:to_lower(X). 368 | 369 | get_content_length([{http_header, _, 'Content-Length', _, V} | _]) -> 370 | list_to_integer(V); 371 | get_content_length([{http_header, _, _X, _, _Y} | T]) -> 372 | get_content_length(T); 373 | get_content_length([]) -> 374 | undefined. 375 | -------------------------------------------------------------------------------- /test/ibrowse_tests.erl: -------------------------------------------------------------------------------- 1 | %%% File : ibrowse_tests.erl 2 | %%% Authors : Benjamin Lee 3 | %%% Dan Schwabe 4 | %%% Brian Richards 5 | %%% Description : Functional tests of the ibrowse library using a live test HTTP server 6 | %%% Created : 18 November 2014 by Benjamin Lee 7 | 8 | -module(ibrowse_tests). 9 | 10 | -include_lib("eunit/include/eunit.hrl"). 11 | -define(PER_TEST_TIMEOUT_SEC, 60). 12 | -define(TIMEDTEST(Desc, Fun), {Desc, {timeout, ?PER_TEST_TIMEOUT_SEC, fun Fun/0}}). 13 | 14 | -define(SERVER_PORT, 8181). 15 | -define(BASE_URL, "http://localhost:" ++ integer_to_list(?SERVER_PORT)). 16 | -define(SHORT_TIMEOUT_MS, 5000). 17 | -define(LONG_TIMEOUT_MS, 30000). 18 | -define(PAUSE_FOR_CONNECTIONS_MS, 4000). 19 | 20 | %%-compile(export_all). 21 | 22 | setup() -> 23 | application:start(crypto), 24 | application:start(public_key), 25 | application:start(ssl), 26 | ibrowse_test_server:start_server(?SERVER_PORT, tcp), 27 | ibrowse:start(), 28 | ok. 29 | 30 | teardown(_) -> 31 | ibrowse:stop(), 32 | ibrowse_test_server:stop_server(?SERVER_PORT), 33 | ok. 34 | 35 | running_server_fixture_test_() -> 36 | {foreach, 37 | fun setup/0, 38 | fun teardown/1, 39 | [ 40 | ?TIMEDTEST("Simple request can be honored", simple_request), 41 | ?TIMEDTEST("Slow server causes timeout", slow_server_timeout), 42 | ?TIMEDTEST("Pipeline depth goes down with responses", pipeline_depth), 43 | ?TIMEDTEST("Pipelines refill", pipeline_refill), 44 | ?TIMEDTEST("Timeout closes pipe", closing_pipes), 45 | ?TIMEDTEST("Requests are balanced over connections", balanced_connections), 46 | ?TIMEDTEST("Pipeline too small signals retries", small_pipeline), 47 | ?TIMEDTEST("Dest status can be gathered", status) 48 | ] 49 | }. 50 | 51 | simple_request() -> 52 | ?assertMatch({ok, "200", _, _}, ibrowse:send_req(?BASE_URL, [], get, [], [])). 53 | 54 | slow_server_timeout() -> 55 | ?assertMatch({error, req_timedout}, ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [], 5000)). 56 | 57 | pipeline_depth() -> 58 | MaxSessions = 2, 59 | MaxPipeline = 2, 60 | RequestsSent = 2, 61 | EmptyPipelineDepth = 0, 62 | 63 | ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()), 64 | 65 | Fun = fun() -> ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end, 66 | times(RequestsSent, fun() -> spawn_link(Fun) end), 67 | 68 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), 69 | 70 | Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()], 71 | ?assertEqual(MaxSessions, length(Counts)), 72 | ?assertEqual(lists:duplicate(MaxSessions, EmptyPipelineDepth), Counts). 73 | 74 | pipeline_refill() -> 75 | MaxSessions = 2, 76 | MaxPipeline = 2, 77 | RequestsToFill = MaxSessions * MaxPipeline, 78 | 79 | %% Send off enough requests to fill sessions and pipelines in rappid succession 80 | Fun = fun() -> ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end, 81 | times(RequestsToFill, fun() -> spawn_link(Fun) end), 82 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), 83 | 84 | % Verify that connections properly reported their completed responses and can still accept more 85 | ?assertMatch({ok, "200", _, _}, ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS)), 86 | 87 | % and do it again to make sure we really are clear 88 | times(RequestsToFill, fun() -> spawn_link(Fun) end), 89 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), 90 | 91 | % Verify that connections properly reported their completed responses and can still accept more 92 | ?assertMatch({ok, "200", _, _}, ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS)). 93 | 94 | closing_pipes() -> 95 | MaxSessions = 2, 96 | MaxPipeline = 2, 97 | RequestsSent = 2, 98 | BalancedNumberOfRequestsPerConnection = 1, 99 | 100 | ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()), 101 | 102 | Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end, 103 | times(RequestsSent, fun() -> spawn_link(Fun) end), 104 | 105 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), 106 | 107 | Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()], 108 | ?assertEqual(MaxSessions, length(Counts)), 109 | ?assertEqual(lists:duplicate(MaxSessions, BalancedNumberOfRequestsPerConnection), Counts), 110 | 111 | timer:sleep(?SHORT_TIMEOUT_MS), 112 | 113 | ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()). 114 | 115 | balanced_connections() -> 116 | MaxSessions = 4, 117 | MaxPipeline = 100, 118 | RequestsSent = 80, 119 | BalancedNumberOfRequestsPerConnection = 20, 120 | 121 | ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()), 122 | 123 | Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?LONG_TIMEOUT_MS) end, 124 | times(RequestsSent, fun() -> spawn_link(Fun) end), 125 | 126 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), 127 | 128 | Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()], 129 | ?assertEqual(MaxSessions, length(Counts)), 130 | 131 | ?assertEqual(lists:duplicate(MaxSessions, BalancedNumberOfRequestsPerConnection), Counts). 132 | 133 | small_pipeline() -> 134 | MaxSessions = 10, 135 | MaxPipeline = 10, 136 | RequestsSent = 100, 137 | FullRequestsPerConnection = 10, 138 | 139 | ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()), 140 | 141 | Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end, 142 | times(RequestsSent, fun() -> spawn(Fun) end), 143 | 144 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), %% Wait for everyone to get in line 145 | 146 | ibrowse:show_dest_status("localhost", 8181), 147 | Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()], 148 | ?assertEqual(MaxSessions, length(Counts)), 149 | 150 | ?assertEqual(lists:duplicate(MaxSessions, FullRequestsPerConnection), Counts), 151 | 152 | Response = ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS), 153 | 154 | ?assertEqual({error, retry_later}, Response). 155 | 156 | status() -> 157 | MaxSessions = 10, 158 | MaxPipeline = 10, 159 | RequestsSent = 100, 160 | 161 | Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end, 162 | times(RequestsSent, fun() -> spawn(Fun) end), 163 | 164 | timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), %% Wait for everyone to get in line 165 | 166 | ibrowse:show_dest_status(), 167 | ibrowse:show_dest_status("http://localhost:8181"). 168 | 169 | 170 | times(0, _) -> 171 | ok; 172 | times(X, Fun) -> 173 | Fun(), 174 | times(X - 1, Fun). 175 | --------------------------------------------------------------------------------