├── .github └── workflows │ └── clojure.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.org ├── changelog.org ├── changelog.org_archive ├── examples ├── body_coercion.clj ├── caching_middleware.clj ├── kubernetes_pod.clj ├── logging-apache-requests.clj └── progress_download.clj ├── project.clj ├── resources └── example-log4j2.properties ├── src └── clj_http │ ├── client.clj │ ├── conn_mgr.clj │ ├── cookies.clj │ ├── core.clj │ ├── core_old.clj │ ├── headers.clj │ ├── links.clj │ ├── multipart.clj │ └── util.clj ├── test-resources ├── big_array_json.json ├── client-keystore ├── keystore ├── m.txt └── small.jpg └── test ├── clj_http └── test │ ├── client_test.clj │ ├── conn_mgr_test.clj │ ├── cookies_test.clj │ ├── core_test.clj │ ├── headers_test.clj │ ├── links_test.clj │ ├── multipart_test.clj │ └── util_test.clj ├── header-html5-test.html ├── header-test.html ├── jetty-logging.properties └── log4j2.properties /.github/workflows/clojure.yml: -------------------------------------------------------------------------------- 1 | name: Clojure CI 2 | 3 | on: 4 | push: 5 | branches: [ 3.x ] 6 | pull_request: 7 | branches: [ 3.x ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | java: ["14", "17", "21"] 15 | clojure: ["1.8", "1.9", "1.10", "1.11"] 16 | 17 | name: Java ${{ matrix.java }} Clojure ${{ matrix.clojure }} 18 | steps: 19 | - uses: actions/checkout@v2 20 | - uses: actions/cache@v2 21 | with: 22 | path: ~/.m2/repository 23 | key: ${{ runner.os }}-lein-${{ hashFiles('**/project.clj') }} 24 | restore-keys: | 25 | ${{ runner.os }}-lein- 26 | 27 | - name: Setup java 28 | uses: actions/setup-java@v1 29 | with: 30 | java-version: ${{ matrix.java }} 31 | 32 | - name: Install dependencies 33 | run: lein deps 34 | 35 | - name: Run tests 36 | run: lein with-profile dev,${{matrix.clojure}} test :all 37 | 38 | - name: Check Reflection Warnings 39 | run: '! lein with-profile dev,${{matrix.clojure}} check 2>&1 | egrep "Reflection warning|Performance warning"' 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # leiningen .gitignore defaults 2 | /target 3 | /classes 4 | /checkouts 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | /.lein-* 10 | /.nrepl-port 11 | 12 | # custom from here on out 13 | build 14 | lib 15 | *.dot 16 | 17 | # use glob syntax. 18 | syntax: glob 19 | creds.clj 20 | Manifest.txt 21 | aws.clj 22 | *.ser 23 | *~ 24 | *.bak 25 | *.off 26 | *.old 27 | .DS_Store 28 | *.#* 29 | *#* 30 | *.classpath 31 | *.project 32 | *.settings 33 | *.pyc 34 | docs/* 35 | doc 36 | http.log 37 | 38 | # Intellij Idea 39 | /*.iml 40 | /.idea 41 | 42 | log/ 43 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | First, thanks for the contributing! Hopefully you find it fairly painless, but 4 | in the interest of explanation, here are some things you might be interested in 5 | when contributing code: 6 | 7 | - Please run the tests locally if you submit a change, you can use `lein all 8 | test :all` to ensure that they pass locally 9 | - If you're able, adding tests with a PR is fantastic! If not, no worries, I can 10 | add those later 11 | - Don't hesitate to ask if you have questions, use `@dakrone` or you can email 12 | me (if it's something you can't talk about publically) at `lee [at] 13 | writequit.org` 14 | 15 | That's it, thanks for using and contributing to clj-http! 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 M. Lee Hinman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /changelog.org: -------------------------------------------------------------------------------- 1 | #+TITLE: clj-http changelog 2 | #+AUTHOR: Lee Hinman 3 | #+STARTUP: fold nodlcheck lognotestate hideall 4 | #+OPTIONS: H:4 num:nil toc:t \n:nil @:t ::t |:t ^:{} -:t f:t *:t 5 | #+OPTIONS: skip:nil d:(HIDE) tags:not-in-toc 6 | #+PROPERTY: header-args :results code :exports both :noweb yes 7 | #+HTML_HEAD: 8 | #+LANGUAGE: en 9 | 10 | * Changelog 11 | List of user-visible changes that have gone into each release 12 | ** 3.12.4 (unreleased) 13 | ** 3.12.3 14 | - Allow http-client re-use in async situation (#599) 15 | https://github.com/dakrone/clj-http/pull/599 16 | ** 3.12.2 17 | - Upgrade Dependencies (#598) 18 | https://github.com/dakrone/clj-http/pull/598 19 | ** 3.12.1 20 | - Bugfix for :normalize-uri (#584) 21 | https://github.com/dakrone/clj-http/pull/584 22 | ** 3.12.0 23 | - Create SSLContext consistently for all connection managers (#575) 24 | https://github.com/dakrone/clj-http/pull/575 25 | - Adds RequestConfig Option :normalize-uri (#583) 26 | https://github.com/dakrone/clj-http/pull/583 27 | ** 3.11.0 28 | - Adds workaround for Async Multipart uploads greater than 25 kb (#574) 29 | https://github.com/dakrone/clj-http/pull/574 30 | - Adds an additional style for multi-param-style added (#562) 31 | https://github.com/dakrone/clj-http/pull/562 32 | - Close transit input stream after reading response (#565) 33 | https://github.com/dakrone/clj-http/pull/565 34 | - Bump patch versions of apache httpcomponents to latest. (#569) 35 | https://github.com/dakrone/clj-http/pull/569 36 | - Fixed decode-json-body (#568) 37 | https://github.com/dakrone/clj-http/pull/568 38 | - Handle quoted parameter values in content type (#573) 39 | https://github.com/dakrone/clj-http/pull/573 40 | ** 3.10.3 41 | - Improve error message when using incompatible version of cheshire 42 | https://github.com/dakrone/clj-http/pull/558 43 | - Properly handle "308 Permanent Redirect" status code 44 | https://github.com/dakrone/clj-http/pull/554 45 | ** 3.10.2 46 | - Fix performance regressions from #528 47 | https://github.com/dakrone/clj-http/pull/546 48 | - Adds support for custom DNS Resolvers 49 | https://github.com/dakrone/clj-http/pull/545 50 | - Buffer :debug output to improve readability 51 | https://github.com/dakrone/clj-http/pull/544 52 | - Improve compatbility with GraalVM 53 | https://github.com/dakrone/clj-http/pull/543 54 | - Bug fix: Check first byte before wrapping response stream with gunzip 55 | https://github.com/dakrone/clj-http/pull/549 56 | ** 3.10.1 57 | - JSON parsing is always strict. See [[file:README.org::*Incrementally%20JSON%20Parsing][README#Incrementally JSON Parsing]]. This is 58 | a *breaking change* and users *must* upgrade to cheshire >= 5.9.0. 59 | https://github.com/dakrone/clj-http/pull/507 60 | ** 3.10.0 61 | - Add trust-manager and key-managers support to the client 62 | https://github.com/dakrone/clj-http/pull/469 63 | - Improving consistency of connection option names 64 | https://github.com/dakrone/clj-http/pull/483 65 | https://github.com/dakrone/clj-http/issues/477 66 | - Ensure Socket Timeout is set for BasicHttpClientConnectionManager 67 | https://github.com/dakrone/clj-http/pull/463 68 | - Reduce body allocation and copying 69 | https://github.com/dakrone/clj-http/pull/475 70 | ** 3.9.1 71 | - Fix body parsing when first byte value is 255 72 | https://github.com/dakrone/clj-http/pull/449 73 | - Add custom =:unexceptional-status= option 74 | https://github.com/dakrone/clj-http/pull/451 75 | ** 3.9.0 76 | - Add support for reusable http clients, returning the client in =:http-client= and allowing one to 77 | be specified (with the same setting) - https://github.com/dakrone/clj-http/issues/441 78 | - Cancelling the =Future= returned from an async http request now also aborts the HttpRequest object 79 | - Async connection managers no longer put the connection manager in an illegal ACTIVE state [[https://github.com/dakrone/clj-http/issues/443][#443]] 80 | - Added the =:cookie-spec= and =:cookie-policy-registry= options for specifying a custom cookie spec 81 | for parsing cookies. Since clj-http doesn't rely on Apache's cookies handling, this is for 82 | advanced users who wish to add their own cookie validation, or use Apache's handling instead of 83 | clj-http's. It also allows a user who wants to registry a custom spec to reuse the spec without 84 | creating it for every request. Semi-related to https://github.com/dakrone/clj-http/issues/444 85 | - Added support for caching HTTP responses from a server. This can dramatically speed up requests to 86 | the same URL. Filling and invalidating the cache is handled by Apache's httpclient-cache project, 87 | with configuration exposed under the =:cache= and =:cache-config= parameters in the option map. 88 | https://github.com/dakrone/clj-http/issues/445 89 | 90 | ** 3.8.0 91 | - Reintroduce the =:save-request= and =:debug-body= options 92 | - +Wrap nested querystring params before form params, fixing 93 | https://github.com/dakrone/clj-http/issues/427+ Reverted, see further below 94 | - Merged https://github.com/dakrone/clj-http/pull/426 to allow an empty SSLGenericSocketFactory 95 | context 96 | - Merged https://github.com/dakrone/clj-http/pull/424 to add :mime-subtype request parameter to 97 | override mime subtype 98 | - create-multipart-entity with three arguments arity lets the selection of =HttpMultipartMode= 99 | - new request key :http-multipart-mode which is HttpMultipartMode/STRICT by default 100 | - Added =:ignore-nested-query-string=, =:flatten-nested-form-params=, and =:flatten-nested-keys= 101 | options for finer-grained control over which nested parts of the request are flattened. Fixes 102 | https://github.com/dakrone/clj-http/issues/427 103 | - Added =:http-builder-fns= and =:async-http-builder-fns= to support arbitrary customizations to the 104 | =HttpClientBuilder= and =HttpAsyncClientBuilder= 105 | - Fixed an issue where redirects to a bad location could cause the async client to hang - 106 | https://github.com/dakrone/clj-http/pull/435 107 | - =client/parse-url= now includes the original URL in the =:url= key 108 | - =core/get-cookie-policy= is now a multimethod. This allows users to customize the return of their 109 | own cookie validation method. 110 | - Empty responses with coercion no longer throw exceptions when processing empty gzipped response 111 | streams. Fixes https://github.com/dakrone/clj-http/issues/257 112 | 113 | ** 3.7.0 114 | This list contains all the changes since 3.0.0. 115 | 116 | Added: 117 | - HttpRequestInterceptor support 155bd23 118 | - protocol-version and reason-phrase f430517 119 | - support for async HTTP requests (like Ring) 44d10ec 120 | - support for different multi-param encoding (:repeating, :array, :indexed) cddeb3e 121 | - Add unparse function aec7dd1 122 | - Added :redirect-strategy :graceful 123 | - Allow RequestConfig and HttpClientContext to be injected feb3c48 124 | 125 | Removed: 126 | - :save-request 127 | 128 | Changed: 129 | - re-written middleware using apache http client 4.5 130 | - Fix retry-handler to be added in correct place a2c31f5 131 | - POST Mutipart: Use charset "UTF-8" instead of "ASCII" as default charset to support internationalization 983508f 132 | 133 | ** 2.0.0 134 | - merged https://github.com/dakrone/clj-http/pull/274 to update Potemkin so it 135 | supports Clojure 1.7.0 correctly 136 | - merged https://github.com/dakrone/clj-http/pull/264 to add support for 137 | coercion of urlencoded data 138 | - make ALL optional dependencies opt-in, rather than opt-out 139 | ** 1.1.2 140 | - bumped dependencies for transit-clj and tools.reader 141 | - merge https://github.com/dakrone/clj-http/pull/263 to only decode body headers 142 | when the content-type is either missing or starts with "text" 143 | ** 1.1.1 144 | - merge https://github.com/dakrone/clj-http/pull/262 to prevent 145 | NullPointerException when decoding body headers with HEAD requests 146 | - merge https://github.com/dakrone/clj-http/pull/261 to decode user info from 147 | URL if provided 148 | - merge https://github.com/dakrone/clj-http/pull/260 to upgrade tools.reader 149 | for better cljs compatibility 150 | - add =304= (not modified) to the list of unexceptional responses, see #259 151 | ** 1.1.0 152 | - merged https://github.com/dakrone/clj-http/pull/255 to add support for Windows 153 | NTLM authentication 154 | - Add the `with-additional-middleware` macro 155 | - Add the ability to specify form-param-encoding for encoding form parameters 156 | - merged https://github.com/dakrone/clj-http/pull/248 to removed deprecated 157 | cookie APIs from cookie.clj 158 | - merged https://github.com/dakrone/clj-http/pull/245 to do some cleanups and 159 | small import fixes 160 | - merged https://github.com/dakrone/clj-http/pull/240 to implement 161 | meta/with-meta for the header map 162 | - merged https://github.com/dakrone/clj-http/pull/242 fixing a connection leak 163 | when http-entity is null 164 | - bumped all dependencies to latest versions 165 | - merged https://github.com/dakrone/clj-http/pull/235 to fix wrap-nested-params 166 | - merged https://github.com/dakrone/clj-http/pull/236 to clean up multipart 167 | constructors and reflection 168 | - merged https://github.com/dakrone/clj-http/pull/234 to allow scheme 169 | customization in default connection 170 | ** 1.0.1 171 | - merged https://github.com/dakrone/clj-http/pull/232 to fix =empty= on 172 | header-map 173 | - fix :json-strict-string-keys 174 | - exclude clojure.core/update from client ns 175 | - added =:decode-cookies= option to allow skipping cookie header decode (if the 176 | server sends incorrectly formatted cookies for some reason) 177 | ** 1.0.0 178 | - merged https://github.com/dakrone/clj-http/pull/215 to add transit support 179 | - drop support for clojure 1.4.0, start testing 1.7.0 180 | - merged https://github.com/dakrone/clj-http/pull/213 to allow passing in an 181 | already existing keystore, not just a path 182 | - merged https://github.com/dakrone/clj-http/pull/211 to detect charset encoding 183 | for url-encode 184 | ** 0.9.2 185 | - merged https://github.com/dakrone/clj-http/pull/206 to handle null passwords 186 | for keystores 187 | - merged https://github.com/dakrone/clj-http/pull/201 to make :auto content type 188 | parsing dispatch pluggable 189 | - Bump crouton and tools.reader dependencies 190 | - Merged https://github.com/dakrone/clj-http/pull/199 to add support for form 191 | parameters in the PATCH method 192 | - Bump dependencies and fix tests for 1.6.0 compatibility 193 | ** 0.9.1 194 | - automatically coerce header values to strings 195 | - fix issue where :ignore-unknown-host wasn't using the =opt= function correctly 196 | ** 0.9.0 197 | - Bumped httpcore to 4.3.2 198 | - Merged https://github.com/dakrone/clj-http/pull/190 to support file multiparts 199 | with content, mime-type and name 200 | - Unify all boolean operators so {:debug true} and {:debug? true} are treated 201 | the same 202 | - Fix :trace-redirects being [nil] when :uri is used 203 | - Merged https://github.com/dakrone/clj-http/pull/184 containing a bevy of 204 | changes: 205 | - initial header-map implementation, allowing headers to be used case 206 | insensitively 207 | - drop support for clojure 1.2 and 1.3 208 | - add support for clojure 1.6 209 | - change all :use statements to :require statements 210 | - use better docstring support for defs 211 | - remove sleep calls in tests 212 | - make Jetty quieter while running tests 213 | - newer type hinting syntax 214 | ** 0.7.9 215 | - Make :decode-body-headers more reliable by using a byte array instead of 216 | slurp. 217 | - Merged https://github.com/dakrone/clj-http/pull/181 to fix some tests 218 | - Merged https://github.com/dakrone/clj-http/pull/178 to eliminate test 219 | reflection 220 | - Merged https://github.com/dakrone/clj-http/pull/177 to update apache HTTP deps 221 | - Merged https://github.com/dakrone/clj-http/pull/175 to add {:as :json-strict} 222 | for output coercion 223 | - Added {:as :json-strict-string-keys} output coercion 224 | - bump dependencies to their latest 225 | - Merged https://github.com/dakrone/clj-http/pull/172 to update .gitignore file 226 | and clean up whitespace for new clojure-mode 227 | - Merged https://github.com/dakrone/clj-http/pull/171 to support SOCKS proxies 228 | * Work log 229 | ** 2015-07-24 230 | - branched master to create 2.x 231 | - start major rewrite on master branch for non-deprecated Apache usage 232 | ** Released 2.0.0 233 | ** 2015-07-18 234 | - merged https://github.com/dakrone/clj-http/pull/274 to update Potemkin so it 235 | supports Clojure 1.7.0 correctly 236 | ** 2015-05-23 237 | - merged https://github.com/dakrone/clj-http/pull/264 to add support for 238 | coercion of urlencoded data 239 | - make ALL optional dependencies opt-in, rather than opt-out 240 | ** Released 1.1.2 241 | ** 2015-05-06 242 | - bumped dependencies for transit-clj and tools.reader 243 | ** 2015-04-24 244 | - merge https://github.com/dakrone/clj-http/pull/263 to only decode body headers 245 | when the content-type is either missing or starts with "text" 246 | ** Released 1.1.1 247 | ** 2015-04-22 248 | - merge https://github.com/dakrone/clj-http/pull/262 to prevent 249 | NullPointerException when decoding body headers with HEAD requests 250 | ** 2015-04-20 251 | - merge https://github.com/dakrone/clj-http/pull/261 to decode user info from 252 | URL if provided 253 | ** 2015-04-14 254 | - merge https://github.com/dakrone/clj-http/pull/260 to upgrade tools.reader 255 | for better cljs compatibility 256 | ** 2015-04-05 257 | - add =304= (not modified) to the list of unexceptional responses, see #259 258 | ** Released 1.1.0 259 | ** 2015-03-03 260 | - merged https://github.com/dakrone/clj-http/pull/255 to add support for Windows 261 | NTLM authentication 262 | ** 2015-02-08 263 | - Add the `with-additional-middleware` macro 264 | - Add the ability to specify form-param-encoding for encoding form parameters 265 | ** 2015-01-19 266 | - merged https://github.com/dakrone/clj-http/pull/248 to removed deprecated 267 | cookie APIs from cookie.clj 268 | - merged https://github.com/dakrone/clj-http/pull/245 to do some cleanups and 269 | small import fixes 270 | ** 2015-01-15 271 | - merged https://github.com/dakrone/clj-http/pull/240 to implement 272 | meta/with-meta for the header map 273 | - merged https://github.com/dakrone/clj-http/pull/242 fixing a connection leak 274 | when http-entity is null 275 | - bumped all dependencies to latest versions 276 | ** 2014-12-13 277 | - merged https://github.com/dakrone/clj-http/pull/235 to fix wrap-nested-params 278 | ** 2014-12-12 279 | - merged https://github.com/dakrone/clj-http/pull/236 to clean up multipart 280 | constructors and reflection 281 | ** 2014-12-02 282 | - merged https://github.com/dakrone/clj-http/pull/234 to allow scheme 283 | customization in default connection 284 | ** Released 1.0.1 285 | ** 2014-10-28 286 | - merged https://github.com/dakrone/clj-http/pull/232 to fix =empty= on 287 | header-map 288 | ** 2014-10-17 289 | - fix :json-strict-string-keys 290 | ** 2014-09-08 291 | - exclude clojure.core/update from client ns 292 | ** 2014-08-15 293 | - added =:decode-cookies= option to allow skipping cookie header decode (if the 294 | server sends incorrectly formatted cookies for some reason) 295 | ** Released 1.0.0 296 | ** 2014-08-11 297 | - merged https://github.com/dakrone/clj-http/pull/215 to add transit support 298 | - drop support for clojure 1.4.0, start testing 1.7.0 299 | ** 2014-08-07 300 | - merged https://github.com/dakrone/clj-http/pull/213 to allow passing in an 301 | already existing keystore, not just a path 302 | ** 2014-07-27 303 | - merged https://github.com/dakrone/clj-http/pull/211 to detect charset encoding 304 | for url-encode 305 | ** Released 0.9.2 306 | ** 2014-05-27 307 | - merged https://github.com/dakrone/clj-http/pull/206 to handle null passwords 308 | for keystores 309 | ** 2014-05-14 310 | - merged https://github.com/dakrone/clj-http/pull/201 to make :auto content type 311 | parsing dispatch pluggable 312 | ** 2014-04-21 313 | - Bump crouton and tools.reader dependencies 314 | ** 2014-04-09 315 | - Merged https://github.com/dakrone/clj-http/pull/199 to add support for form 316 | parameters in the PATCH method 317 | ** 2014-03-26 318 | - Bump dependencies and fix tests for 1.6.0 compatibility 319 | ** Released 0.9.1 320 | ** 2014-03-15 321 | - automatically coerce header values to strings 322 | ** 2014-03-05 323 | - fix issue where :ignore-unknown-host wasn't using the =opt= function correctly 324 | ** Released 0.9.0 325 | ** 2014-02-25 326 | - Bumped httpcore to 4.3.2 327 | ** 2014-02-19 328 | - Merged https://github.com/dakrone/clj-http/pull/190 to support file multiparts 329 | with content, mime-type and name 330 | ** 2014-02-16 331 | - Unify all boolean operators so {:debug true} and {:debug? true} are treated 332 | the same 333 | ** 2014-02-09 334 | - Fix :trace-redirects being [nil] when :uri is used 335 | ** 2014-02-06 336 | - Merged https://github.com/dakrone/clj-http/pull/184 containing a bevy of 337 | changes: 338 | - initial header-map implementation, allowing headers to be used case 339 | insensitively 340 | - drop support for clojure 1.2 and 1.3 341 | - add support for clojure 1.6 342 | - change all :use statements to :require statements 343 | - use better docstring support for defs 344 | - remove sleep calls in tests 345 | - make Jetty quieter while running tests 346 | - newer type hinting syntax 347 | ** Released 0.7.9 348 | ** 2014-02-01 349 | - Make :decode-body-headers more reliable by using a byte array instead of 350 | slurp. 351 | -------------------------------------------------------------------------------- /changelog.org_archive: -------------------------------------------------------------------------------- 1 | 2 | Archived entries from file /Users/hinmanm/src/clj/clj-http/changelog.org 3 | 4 | 5 | * Archived Tasks 6 | 7 | ** 0.7.8 8 | :PROPERTIES: 9 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 10 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 11 | :ARCHIVE_OLPATH: Changelog 12 | :ARCHIVE_CATEGORY: changelog 13 | :END: 14 | - Added the `proxy-ignore-hosts` option to allow specifying a list 15 | of hosts where a proxy should be ignored 16 | - merged https://github.com/dakrone/clj-http/pull/166 to fix some 17 | small whitespace and reflection stuff 18 | - Close the body of a response in wrap-redirects since all bodies 19 | are streams now. Otherwise, the body is kept open indefinitely. 20 | 21 | ** 0.7.7 22 | :PROPERTIES: 23 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 24 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 25 | :ARCHIVE_OLPATH: Changelog 26 | :ARCHIVE_CATEGORY: changelog 27 | :END: 28 | - merged https://github.com/dakrone/clj-http/pull/162 to pass 29 | through json opts for form-param encoding 30 | - bumped dependencies 31 | - fix #159 - issue with :decode-body-headers introduced with 32 | streaming bodies 33 | - merged https://github.com/dakrone/clj-http/pull/156 to add 34 | `:raw-headers` option to return an additional 35 | untouched :raw-headers map 36 | - merged https://github.com/dakrone/clj-http/pull/154 to handle 37 | query-params not clobbering query-params in the URL string 38 | - bump main deps 39 | - merged https://github.com/dakrone/clj-http/pull/151 to prevent 40 | shutting down a reusable connection manager when an error occurs 41 | 42 | ** 0.7.6 43 | :PROPERTIES: 44 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 45 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 46 | :ARCHIVE_OLPATH: Changelog 47 | :ARCHIVE_CATEGORY: changelog 48 | :END: 49 | - add logging config for local testing only 50 | - remove "content-encoding" header if the body is automatically 51 | decompressed to allow for pass-through. If header is removed, 52 | assoc :orig-content-encoding to response map. 53 | - merged https://github.com/dakrone/clj-http/pull/149 to fix 54 | closing the stream when coerced to byte array 55 | - merged https://github.com/dakrone/clj-http/pull/146 to correctly 56 | reference parameter names 57 | 58 | ** 0.7.5 59 | :PROPERTIES: 60 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 61 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 62 | :ARCHIVE_OLPATH: Changelog 63 | :ARCHIVE_CATEGORY: changelog 64 | :END: 65 | - Only redirect if a "location" header is actually, present, avoiding an 66 | NPE in the event it's missing. (fixes #145) 67 | 68 | ** 0.7.4 69 | :PROPERTIES: 70 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 71 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 72 | :ARCHIVE_OLPATH: Changelog 73 | :ARCHIVE_CATEGORY: changelog 74 | :END: 75 | - merged https://github.com/dakrone/clj-http/pull/143 for fixing some 76 | weirdness around body streams and inflation 77 | - streams everywhere, all bodies coming out of core.clj are now streams, so 78 | {:as :stream} truly streams the response, keeping it out of memory 79 | - remove some more reflection 80 | 81 | ** 0.7.3 82 | :PROPERTIES: 83 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 84 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 85 | :ARCHIVE_OLPATH: Changelog 86 | :ARCHIVE_CATEGORY: changelog 87 | :END: 88 | - correctly close single client connection manager if {:as :stream} is used, fixes #142 89 | - merged https://github.com/dakrone/clj-http/pull/138 to preserve 90 | http method for 307 redirect 91 | - merged in parse-url parameters into follow-redirect so request 92 | map is not inconsistent 93 | - bumped http* deps to 4.2.5 94 | - fixed cookie compact-map not to remove falsey values, only nil 95 | ones 96 | - merged https://github.com/dakrone/clj-http/pull/135 to fix 97 | discard always defaulting to true 98 | - add *current-middleware* to see available middleware during a 99 | with-middleware request (for nesting) 100 | 101 | ** 0.7.2 102 | :PROPERTIES: 103 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 104 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 105 | :ARCHIVE_OLPATH: Changelog 106 | :ARCHIVE_CATEGORY: changelog 107 | :END: 108 | - merged https://github.com/dakrone/clj-http/pull/127 to allow 109 | custom cookie policies 110 | - allow specifying :length for mulitpart inputstream bodies to 111 | avoid chunked transfer encoding 112 | - bumped cheshire to 5.1.1 113 | - merged https://github.com/dakrone/clj-http/pull/133 to remove 114 | some reflection 115 | 116 | ** 0.7.1 117 | :PROPERTIES: 118 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 119 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 120 | :ARCHIVE_OLPATH: Changelog 121 | :ARCHIVE_CATEGORY: changelog 122 | :END: 123 | - clarify :throw-exceptions in documentation 124 | - define default-middleware for use in wrap-request, remove bad 125 | all-middleware method 126 | - merged https://github.com/dakrone/clj-http/pull/130 to encode 127 | query-params 128 | - merged https://github.com/dakrone/clj-http/pull/124 to handle 129 | URL-encoding invalid characters in the URI 130 | - bump cheshire to 5.1.0 131 | - Switch from deprecated SingleClientConnManager to BasicClientConnectionManager 132 | - merged https://github.com/dakrone/clj-http/pull/126 to bump 133 | httpcore version 134 | - bump dependencies to latest versions 135 | 136 | ** 0.7.0 137 | :PROPERTIES: 138 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 139 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 140 | :ARCHIVE_OLPATH: Changelog 141 | :ARCHIVE_CATEGORY: changelog 142 | :END: 143 | - merged https://github.com/dakrone/clj-http/pull/122 for 144 | using *data-readers* when using tools.reader to parse EDN 145 | - fix an issue with 1.3 where *data-readers* is not available 146 | - merged https://github.com/dakrone/clj-http/pull/121 to fix 147 | auto-coercion with json 148 | - support application/edn as an auto-coercion type 149 | - add tools.reader as an optional dependency, edn/read will be 150 | used if available, otherwise read-string with *read-eval* bound 151 | to false is used. See https://github.com/dakrone/clj-http/pull/120 152 | - Bump clojure to 1.5.1 153 | 154 | ** 0.6.5 155 | :PROPERTIES: 156 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 157 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 158 | :ARCHIVE_OLPATH: Changelog 159 | :ARCHIVE_CATEGORY: changelog 160 | :END: 161 | - allow json coercion for exception cases based on :coerce setting, 162 | can be either :always, :exceptional or :unexceptional 163 | - Update clojure to 1.5 164 | - Move SingleClientConnManager shutdown into finally block 165 | - bind *read-eval* to false when reading for {:as :clojure} 166 | - bump cheshire to 5.0.2 167 | 168 | ** 0.6.4 169 | :PROPERTIES: 170 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 171 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 172 | :ARCHIVE_OLPATH: Changelog 173 | :ARCHIVE_CATEGORY: changelog 174 | :END: 175 | - merged https://github.com/dakrone/clj-http/pull/113 to update 176 | the connection pooling code 177 | - refactor pooled connection managers to allow specifying 178 | the :connection-manager option 179 | - merged https://github.com/dakrone/clj-http/pull/112 to allow 180 | json coercion on error responses when :as :auto is used 181 | - allow redirects when :url is not set in the request 182 | - merged https://github.com/dakrone/clj-http/pull/110 to handle the 183 | case when the server-side uses deflate incorrectly 184 | - added `with-middleware` to allow running requests with a custom 185 | middleware list 186 | - added `all-middleware` var listing all the wrap-* middleware that 187 | clj-http knows of 188 | - clj-http.client/request is now marked as dynamic for rebinding 189 | 190 | ** 0.6.3 191 | :PROPERTIES: 192 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 193 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 194 | :ARCHIVE_OLPATH: Changelog 195 | :ARCHIVE_CATEGORY: changelog 196 | :END: 197 | - Remove wrap-cookie-store middleware, CookieStore headers are 198 | automatically added by Apache 199 | - set the SINGLE_COOKIE_HEADER value to true to ensure Apache sends 200 | only one "Cookie:" header 201 | - Do not add CookieStore or Cookie header if there are no cookies 202 | in the cookie jar 203 | 204 | ** 0.6.2 205 | :PROPERTIES: 206 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 207 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 208 | :ARCHIVE_OLPATH: Changelog 209 | :ARCHIVE_CATEGORY: changelog 210 | :END: 211 | - merged https://github.com/dakrone/clj-http/pull/106 to remove 212 | query params for redirection. 213 | - whitespace fixes; fix test that wasn't working correctly 214 | 215 | ** 0.6.1 216 | :PROPERTIES: 217 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 218 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 219 | :ARCHIVE_OLPATH: Changelog 220 | :ARCHIVE_CATEGORY: changelog 221 | :END: 222 | - bump httpcore to 4.2.3 223 | - Fix an issue (#105) related to the "Content-Length" header being 224 | automatically added to GET requests 225 | 226 | ** 0.6.0 227 | :PROPERTIES: 228 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 229 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 230 | :ARCHIVE_OLPATH: Changelog 231 | :ARCHIVE_CATEGORY: changelog 232 | :END: 233 | (bumped to 0.6.0 since Cheshire has changed major versions) 234 | - Update Cheshire to 5.0.1 235 | - Add type hint for getting headers from body (michaelklishin) 236 | 237 | ** 0.5.8 238 | :PROPERTIES: 239 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 240 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 241 | :ARCHIVE_OLPATH: Changelog 242 | :ARCHIVE_CATEGORY: changelog 243 | :END: 244 | - add buffering for HttpEntity, with ability to turn off if needed, 245 | fixes lein issue with repeatable requests 246 | 247 | ** 0.5.7 248 | :PROPERTIES: 249 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 250 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 251 | :ARCHIVE_OLPATH: Changelog 252 | :ARCHIVE_CATEGORY: changelog 253 | :END: 254 | - create a custom X509HostnameVerifier for the :insecure? option 255 | - explicitly require httpcore instead of leaving it to a transitive dep 256 | - update httpcomponents to 4.2.2 257 | - implement HTML5 charset header reading from body 258 | 259 | ** 0.5.6 260 | :PROPERTIES: 261 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 262 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 263 | :ARCHIVE_OLPATH: Changelog 264 | :ARCHIVE_CATEGORY: changelog 265 | :END: 266 | - bump Crouton to 0.1.1 for faster speeds 267 | - add feature to decode body headers, merging them into response 268 | headers if they are present 269 | - merged https://github.com/dakrone/clj-http/pull/98 to add 270 | optional :default-per-route to with-connection-pool 271 | 272 | ** 0.5.5 273 | :PROPERTIES: 274 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 275 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 276 | :ARCHIVE_OLPATH: Changelog 277 | :ARCHIVE_CATEGORY: changelog 278 | :END: 279 | - bump cheshire to fix json encoding bug 280 | 281 | ** 0.5.4 282 | :PROPERTIES: 283 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 284 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 285 | :ARCHIVE_OLPATH: Changelog 286 | :ARCHIVE_CATEGORY: changelog 287 | :END: 288 | - merged https://github.com/dakrone/clj-http/pull/95 to add support 289 | for setting aribtrary client params to the http client 290 | - Merged https://github.com/dakrone/clj-http/pull/94 to remove some 291 | reflection 292 | - update cheshire dep, make clojure a dev-dependency 293 | - allow overriding the multipart part name with :part-name 294 | 295 | ** 0.5.3 296 | :PROPERTIES: 297 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 298 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 299 | :ARCHIVE_OLPATH: Changelog 300 | :ARCHIVE_CATEGORY: changelog 301 | :END: 302 | - merged https://github.com/dakrone/clj-http/pull/91 to add support 303 | for :digest-auth 304 | - added request timing middleware to add :request-time key for 305 | request timing 306 | - add wrap-cookie-store to send cookie-store cookies with a request 307 | automatically 308 | - merged https://github.com/dakrone/clj-http/pull/90 to standardize 309 | on lower-case headers for HTTP requests 310 | 311 | ** 0.5.2 312 | :PROPERTIES: 313 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 314 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 315 | :ARCHIVE_OLPATH: Changelog 316 | :ARCHIVE_CATEGORY: changelog 317 | :END: 318 | - merged https://github.com/dakrone/clj-http/pull/88 to add chunked encoding 319 | support (=:length= no longer required along with input stream =:body=) 320 | 321 | ** 0.5.1 322 | :PROPERTIES: 323 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 324 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 325 | :ARCHIVE_OLPATH: Changelog 326 | :ARCHIVE_CATEGORY: changelog 327 | :END: 328 | - fix clojure 1.3's exception wrapping for some exceptions 329 | - merged https://github.com/dakrone/clj-http/pull/87 to allow using 330 | http.nonProxyHosts 331 | - mark json-encode and json-decode dynamic, so they could be 332 | rebound if desired 333 | - update httpclient and httpmime to 4.2.1 334 | - update commons-codec to 1.6 335 | - update common-io to 2.4 336 | - change body decompression to be optional, if desired 337 | - make the :content-type and :character-encoding options part of 338 | middleware, not the core request 339 | - document all the middleware 340 | - merged https://github.com/dakrone/clj-http/pull/85 to allow 341 | low-level callback for debugging 342 | 343 | ** 0.5.0 344 | :PROPERTIES: 345 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 346 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 347 | :ARCHIVE_OLPATH: Changelog 348 | :ARCHIVE_CATEGORY: changelog 349 | :END: 350 | - rewrite multipart body entity creation to use different map 351 | format, allowing :mime-type and :encoding keys in some cases 352 | 353 | ** 0.4.4 354 | :PROPERTIES: 355 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 356 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 357 | :ARCHIVE_OLPATH: Changelog 358 | :ARCHIVE_CATEGORY: changelog 359 | :END: 360 | - bump cheshire to 4.0.1 and slingshot to 0.10.3 361 | - fix an issue where cookies were encoded and should not be 362 | - merged https://github.com/dakrone/clj-http/pull/80 to allow 363 | specifying the keystore type 364 | - merged https://github.com/dakrone/clj-http/pull/79 to allow 365 | pluggable output coercion (multimethod) 366 | 367 | ** 0.4.3 368 | :PROPERTIES: 369 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 370 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 371 | :ARCHIVE_OLPATH: Changelog 372 | :ARCHIVE_CATEGORY: changelog 373 | :END: 374 | - support custom x509 keystore/trust-stores 375 | 376 | ** 0.4.2 377 | :PROPERTIES: 378 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 379 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 380 | :ARCHIVE_OLPATH: Changelog 381 | :ARCHIVE_CATEGORY: changelog 382 | :END: 383 | - fixed an issue where multiple link headers would cause an 384 | exception to be thrown 385 | 386 | ** 0.4.1 387 | :PROPERTIES: 388 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 389 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 390 | :ARCHIVE_OLPATH: Changelog 391 | :ARCHIVE_CATEGORY: changelog 392 | :END: 393 | - added :debug-body that adds plaintext body information to 394 | the :debug output 395 | - fix json encoded form params with nested maps 396 | - fix attempted json coercion when a bad status is received 397 | - merged https://github.com/dakrone/clj-http/pull/69 to add support 398 | for :oauth-token authentication 399 | - merged https://github.com/dakrone/clj-http/pull/70 to save the 400 | apache Http object when :save-request? is true 401 | - merged https://github.com/dakrone/clj-http/pull/68 to support 402 | additional options/delete/copy/move HTTP methods 403 | - add support for the :patch method type 404 | 405 | ** 0.4.0 406 | :PROPERTIES: 407 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 408 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 409 | :ARCHIVE_OLPATH: Changelog 410 | :ARCHIVE_CATEGORY: changelog 411 | :END: 412 | - merged https://github.com/dakrone/clj-http/pull/66 to add support 413 | for 'Link' header 414 | - added ability to specify your own retry-handler for IOExceptions 415 | if desired 416 | - bumped httpclient and httpmime to 4.1.3 417 | - bump to released version of clojure (1.4) 418 | - added documentation about ipv6 requests 419 | - fixed https://github.com/dakrone/clj-http/issues/57 by have 420 | wrap-redirects redirect according to the RFC and adding 421 | the :force-redirects option to be more browser-like 422 | - merged https://github.com/dakrone/clj-http/pull/61 to add support 423 | for nested param maps 424 | 425 | ** 0.3.6 426 | :PROPERTIES: 427 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 428 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 429 | :ARCHIVE_OLPATH: Changelog 430 | :ARCHIVE_CATEGORY: changelog 431 | :END: 432 | - fixed an issue where urls like http://user:pass@foo.com didn't 433 | work correctly for basic-auth 434 | - added support for cookie stores 435 | - added utility methods to retrieve cookies as a map from the 436 | cookie store 437 | - set the default maximum number of redirects to 20 438 | 439 | ** 0.3.5 440 | :PROPERTIES: 441 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 442 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 443 | :ARCHIVE_OLPATH: Changelog 444 | :ARCHIVE_CATEGORY: changelog 445 | :END: 446 | - same as 0.3.4, but with a newer cheshire that doesn't interfere 447 | with clj-json 448 | 449 | ** 0.3.4 450 | :PROPERTIES: 451 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 452 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 453 | :ARCHIVE_OLPATH: Changelog 454 | :ARCHIVE_CATEGORY: changelog 455 | :END: 456 | - improved commit from pull/55 to make the predicate more generalized to 457 | any kind of entity request 458 | - make Cheshire an optional dependency, only for {:as :json} and 459 | json form-params 460 | - merged https://github.com/dakrone/clj-http/pull/55 to fix HEAD 461 | requests with body contents 462 | - merged https://github.com/dakrone/clj-http/pull/53 to add status 463 | functions into the clj-http.client namespace 464 | - added the ability to specify {:as :clojure} to get back a clojure 465 | datastructure, or {:as :auto} with content-type=application/clojure 466 | - merged https://github.com/dakrone/clj-http/pull/52 to support 467 | json-encoded form params 468 | - added a test for json-encoded form params as request body 469 | 470 | ** 0.3.3 471 | :PROPERTIES: 472 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 473 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 474 | :ARCHIVE_OLPATH: Changelog 475 | :ARCHIVE_CATEGORY: changelog 476 | :END: 477 | - merged https://github.com/dakrone/clj-http/pull/51 to 478 | allow :form-params on PUT requests 479 | - bump Cheshire and slingshot deps 480 | - add the :throw-entire-message? option to include resp in 481 | Exception message 482 | - throw an IllegalArgumentException instead of a regulor Exception 483 | on nil urls 484 | - add ability to redirect to relative paths (ngrunwald) 485 | 486 | ** 0.3.2 487 | :PROPERTIES: 488 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 489 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 490 | :ARCHIVE_OLPATH: Changelog 491 | :ARCHIVE_CATEGORY: changelog 492 | :END: 493 | - merged https://github.com/dakrone/clj-http/pull/48 to fix :stream 494 | bodies (to make sure they are not coerced on output) 495 | - merged https://github.com/dakrone/clj-http/pull/49 to check for 496 | nil URLs when using client functions 497 | - switch from assertions to exceptions for nil URLs 498 | - merged https://github.com/dakrone/clj-http/pull/46 to 499 | add :trace-redirects to the response map 500 | - merged https://github.com/dakrone/clj-http/pull/47 to allow GET 501 | requests with a :body set 502 | - merged https://github.com/dakrone/clj-http/pull/44 to add ability 503 | to specify maximum number of redirects 504 | - add tests for max-redirects 505 | - merged https://github.com/dakrone/clj-http/pull/42 to allow 506 | strings or keywords for :scheme in requests 507 | - added test for different :schemes 508 | 509 | ** 0.3.1 510 | :PROPERTIES: 511 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 512 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 513 | :ARCHIVE_OLPATH: Changelog 514 | :ARCHIVE_CATEGORY: changelog 515 | :END: 516 | - merged https://github.com/dakrone/clj-http/pull/40 to allow 517 | per-request proxy settings 518 | - remove a few more reflections 519 | - added ablity to return the body as a stream with {:as :stream} 520 | - general code cleanup 521 | 522 | ** 0.3.0 523 | :PROPERTIES: 524 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 525 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 526 | :ARCHIVE_OLPATH: Changelog 527 | :ARCHIVE_CATEGORY: changelog 528 | :END: 529 | - add ability to ignore unknown host if desired ({:ignore-unknown-host? true}) 530 | - use much better Enitity's for the body, depending on type 531 | - bump all dependencies 532 | - test re-org to make better sense (and allow C-c t in emacs) 533 | - merged https://github.com/dakrone/clj-http/pull/36 to fix 534 | url-encoding of multiple query params using the same key 535 | - merged https://github.com/dakrone/clj-http/pull/34 to fix 536 | decoding cookies that don't follow RFC spec 537 | - Add better coercion, adding {:as :json}, {:as :json-string-keys} 538 | and {:as :auto} 539 | 540 | ** 0.2.7 541 | :PROPERTIES: 542 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 543 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 544 | :ARCHIVE_OLPATH: Changelog 545 | :ARCHIVE_CATEGORY: changelog 546 | :END: 547 | - merged https://github.com/dakrone/clj-http/pull/31 to remove more 548 | reflection warnings 549 | - some whitespace changes 550 | - merged https://github.com/dakrone/clj-http/pull/30 to remove more 551 | reflection warnings 552 | - removed swank from dev deps 553 | - bump 1.4 to alpha3 in multi deps 554 | 555 | ** 0.2.6 556 | :PROPERTIES: 557 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 558 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 559 | :ARCHIVE_OLPATH: Changelog 560 | :ARCHIVE_CATEGORY: changelog 561 | :END: 562 | - don't use :server-port unless required (fixes problem with some 563 | web servers) 564 | - smaller error message on exceptions (thrown object is still the same) 565 | - added the :save-request? option to return the request object in 566 | a :request key in the response map 567 | - multiple headers with the same name are now preserved when they 568 | have differing cases 569 | 570 | ** 0.2.5 571 | :PROPERTIES: 572 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 573 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 574 | :ARCHIVE_OLPATH: Changelog 575 | :ARCHIVE_CATEGORY: changelog 576 | :END: 577 | - multipart form uploads 578 | - bump slingshot to 0.9.0 579 | 580 | ** 0.2.4 581 | :PROPERTIES: 582 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 583 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 584 | :ARCHIVE_OLPATH: Changelog 585 | :ARCHIVE_CATEGORY: changelog 586 | :END: 587 | - Got a functioning reusable connection method, 588 | (with-connection-pool ...) 589 | - upgrade slingshot to 0.8.0 590 | - upgrade commons-io to 2.1 591 | - merged https://github.com/dakrone/clj-http/pull/20 to 592 | allow :basic-auth as a string 593 | 594 | ** 0.2.3 595 | :PROPERTIES: 596 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 597 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 598 | :ARCHIVE_OLPATH: Changelog 599 | :ARCHIVE_CATEGORY: changelog 600 | :END: 601 | - added :insecure? flag 602 | - fix AOT by requiring clojure.pprint 603 | - wrap-redirects now handles recursive redirects 604 | 605 | ** 0.2.2 606 | :PROPERTIES: 607 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 608 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 609 | :ARCHIVE_OLPATH: Changelog 610 | :ARCHIVE_CATEGORY: changelog 611 | :END: 612 | - wrap-exceptions now uses Slingshot to throw a much more useful 613 | exception when there was a problem with the request 614 | - fixed an issue when malformed server responses could NPE the 615 | decompression middleware 616 | - added a :debug flag to pretty-print the request map and object 617 | to stdout before performing the request to aid in debugging 618 | 619 | ** 0.2.1 620 | :PROPERTIES: 621 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 622 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 623 | :ARCHIVE_OLPATH: Changelog 624 | :ARCHIVE_CATEGORY: changelog 625 | :END: 626 | - decode cookies from response into :cookies (thanks r0man) 627 | - redone redirects, they can now be toggled with {:follow-redirects 628 | false} in the request 629 | - decompression of responses has been fixed (thanks senior) 630 | - accept Content-Encoding or content-encoding from responses 631 | (thanks senior) 632 | - added ability to specify sending a url-encoded :body of form 633 | params using {:form-params {:key value}} (thanks senior) 634 | 635 | ** 0.2.0 636 | :PROPERTIES: 637 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 638 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 639 | :ARCHIVE_OLPATH: Changelog 640 | :ARCHIVE_CATEGORY: changelog 641 | :END: 642 | - updated dependencies to be the latest versions 643 | - added ability to use system proxy for connections (thanks jou4) 644 | - added ability to specify socket and connection timeouts in 645 | request (thanks zkim) 646 | 647 | ** 0.1.3 648 | :PROPERTIES: 649 | :ARCHIVE_TIME: 2014-03-05 Wed 16:32 650 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 651 | :ARCHIVE_OLPATH: Changelog 652 | :ARCHIVE_CATEGORY: changelog 653 | :END: 654 | - see: https://github.com/mmcgrana/clj-http 655 | 656 | ** Released 0.7.8 657 | :PROPERTIES: 658 | :ARCHIVE_TIME: 2015-04-22 Wed 23:13 659 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 660 | :ARCHIVE_OLPATH: Work log 661 | :ARCHIVE_CATEGORY: changelog 662 | :END: 663 | 664 | ** 2014-01-03 665 | :PROPERTIES: 666 | :ARCHIVE_TIME: 2015-04-22 Wed 23:13 667 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 668 | :ARCHIVE_OLPATH: Work log 669 | :ARCHIVE_CATEGORY: changelog 670 | :END: 671 | - bump dependencies to their latest 672 | - Merged https://github.com/dakrone/clj-http/pull/172 to update .gitignore file 673 | and clean up whitespace for new clojure-mode 674 | - Merged https://github.com/dakrone/clj-http/pull/171 to support SOCKS proxies 675 | 676 | ** 2014-01-15 677 | :PROPERTIES: 678 | :ARCHIVE_TIME: 2015-04-22 Wed 23:13 679 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 680 | :ARCHIVE_OLPATH: Work log 681 | :ARCHIVE_CATEGORY: changelog 682 | :END: 683 | - Merged https://github.com/dakrone/clj-http/pull/175 to add {:as :json-strict} 684 | for output coercion 685 | - Added {:as :json-strict-string-keys} output coercion 686 | 687 | ** 2014-01-21 688 | :PROPERTIES: 689 | :ARCHIVE_TIME: 2015-04-22 Wed 23:13 690 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 691 | :ARCHIVE_OLPATH: Work log 692 | :ARCHIVE_CATEGORY: changelog 693 | :END: 694 | - Merged https://github.com/dakrone/clj-http/pull/177 to update apache HTTP deps 695 | 696 | ** 2014-01-27 697 | :PROPERTIES: 698 | :ARCHIVE_TIME: 2015-04-22 Wed 23:13 699 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 700 | :ARCHIVE_OLPATH: Work log 701 | :ARCHIVE_CATEGORY: changelog 702 | :END: 703 | - Merged https://github.com/dakrone/clj-http/pull/178 to eliminate test 704 | reflection 705 | 706 | ** 2014-01-28 707 | :PROPERTIES: 708 | :ARCHIVE_TIME: 2015-04-22 Wed 23:13 709 | :ARCHIVE_FILE: ~/src/clj/clj-http/changelog.org 710 | :ARCHIVE_OLPATH: Work log 711 | :ARCHIVE_CATEGORY: changelog 712 | :END: 713 | - Merged https://github.com/dakrone/clj-http/pull/181 to fix some tests 714 | -------------------------------------------------------------------------------- /examples/body_coercion.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.examples.body-coercion 2 | (:require [clj-http.client :as http] 3 | [camel-snake-kebab.core :refer [->kebab-case-keyword]])) 4 | 5 | ;; register your own body coercers by participating in the coerce-response-body multimethod 6 | ;; dispatch to it by using {:as :json-kebab-keys} as an argument to http client calls 7 | 8 | ;; this example uses camel-snake-kebab to turn a camel-cased JSON API into 9 | ;; idiomatic kebab-cased keywords in clojure data structures and is much 10 | ;; faster than applying via postwalk or similar 11 | 12 | (defmethod http/coerce-response-body :json-kebab-keys [req resp] 13 | (http/coerce-json-body req resp (memoize ->kebab-case-keyword) false)) 14 | 15 | ;; example of use; note that in the response, the first field is called userId 16 | ;; 17 | ;; (:body (http/get "http://jsonplaceholder.typicode.com/posts/1" {:as :json-kebab-keys})) 18 | ;; => 19 | ;; {:user-id 1, 20 | ;; :id 1, 21 | ;; :title "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", 22 | ;; :body "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" 23 | ;; } 24 | -------------------------------------------------------------------------------- /examples/caching_middleware.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.examples.caching-middleware 2 | "Example middleware that caches successful GET requests using core.cache." 3 | (:require 4 | [clj-http.client :as http] 5 | [clojure.core.cache :as cache]) 6 | (:import 7 | (java.nio.charset StandardCharsets))) 8 | 9 | (def http-cache (atom (cache/ttl-cache-factory {} :ttl (* 60 60 1000)))) 10 | 11 | (defn slurp-bytes 12 | "Read all bytes from the stream. 13 | Use for example when the bytes will be in demand after stream has been closed." 14 | [stream] 15 | (.getBytes (slurp stream) StandardCharsets/UTF_8)) 16 | 17 | (defn- cached-response 18 | "Look up the response in the cache using URL as the cache key. 19 | If the cache has the response, return the cached value. 20 | If the cache does not have the response, invoke the remaining middleware functions 21 | to perform the request and receive the response. 22 | If the response is successful (2xx) and is a GET, store the response in the cache. 23 | Return the response." 24 | ([client req] 25 | (let [cache-key (str (:server-name req) (:uri req) "?" (:query-string req))] 26 | (if (cache/has? @http-cache cache-key) 27 | (do 28 | (println "CACHE HIT") 29 | (reset! http-cache (cache/hit @http-cache cache-key)) ; update cache stats 30 | (cache/lookup @http-cache cache-key)) ; return cached value 31 | ; do not invoke further middleware 32 | (do 33 | (println "CACHE MISS") 34 | (let [resp (update (client req) :body slurp-bytes)] ; middleware chain invoked 35 | (when (and (http/success? resp) (= (:request-method req) :get)) 36 | (reset! http-cache (cache/miss @http-cache cache-key resp)) ; update cache value 37 | resp))))))) 38 | 39 | (defn wrap-caching-middleware 40 | "Middleware are functions that add functionality to handlers. 41 | The argument client is a handler. 42 | This wrapper function adds response caching to the client." 43 | [client] 44 | (fn 45 | ([req] 46 | (cached-response client req)))) 47 | 48 | (defn example 49 | "Add the caching middleware and perform a GET request using the URI argument. 50 | Subsequent invocations of this function using an identical URI argument 51 | before the Time To Live expires can be expected to hit the cache." 52 | [& uri] 53 | (-> (time (http/with-additional-middleware [#'wrap-caching-middleware] 54 | (http/get (or uri "https://api.github.com") 55 | { 56 | ;; :debug true 57 | ;; :debug-body true 58 | ;; :throw-entire-message? true 59 | }))) 60 | (select-keys ,,, [:status :reason-phrase :headers]))) 61 | 62 | ;; Try this out: 63 | ;; 64 | ;; user> (use '[clj-http.examples.caching-middleware :as mw]) 65 | ;; nil 66 | ;; user> (mw/example) 67 | ;; CACHE MISS 68 | ;; "Elapsed time: 1910.027361 msecs" 69 | ;; {:status 200, :reason-phrase "OK"} 70 | ;; user> (mw/example) 71 | ;; CACHE HIT 72 | ;; "Elapsed time: 0.83484 msecs" 73 | ;; {:status 200, :reason-phrase "OK"} 74 | ;; user> 75 | -------------------------------------------------------------------------------- /examples/kubernetes_pod.clj: -------------------------------------------------------------------------------- 1 | (:ns clj-http.examples.kubernetes-pod 2 | "This is an example of calling the Kubernetes API from inside a pod. K8s uses a 3 | custom CA so that you can authenticate the API server, and provides a token per pod 4 | so that each pod can authenticate itself with the APi server. 5 | 6 | If you are still having 401/403 errors, look carefully at the message, if it includes 7 | a ServiceAccount name, this part worked, and your problem is likely at the Role/RoleBinding level." 8 | (:require [clj-http.client :as http] 9 | [less.awful.ssl :refer [trust-store]])) 10 | 11 | ;; Note that this is not a working example, you'll need to figure out your K8s API path. 12 | (let [k8s-trust-store (trust-store (clojure.java.io/file "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")) 13 | bearer-token (format "Bearer %s" (slurp "/var/run/secrets/kubernetes.io/serviceaccount/token")) 14 | kube-api-host (System/getenv "KUBERNETES_SERVICE_HOST") 15 | kube-api-port (System/getenv "KUBERNETES_SERVICE_PORT")] 16 | (http/get 17 | (format "https://%s:%s/apis/" kube-api-host kube-api-port) 18 | {:trust-store k8s-trust-store 19 | :headers {:authorization bearer-token}})) 20 | 21 | -------------------------------------------------------------------------------- /examples/logging-apache-requests.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.examples.logging-apache-requests 2 | "This is an example of configuring Apache's log4j2 logging from Clojure, so 3 | that the http client logging can be seen" 4 | (:require [clj-http.client :as http]) 5 | (:import (org.apache.logging.log4j Level 6 | LogManager))) 7 | 8 | ;; This is a helper function to change the log level for log4j2. If you use a 9 | ;; different logging framework (and subsequently a different bridge for log4j 10 | ;; then you'll need to substitute your own logging configuration 11 | (defn change-log-level! [logger-name level] 12 | (let [ctx (LogManager/getContext false) 13 | config (.getConfiguration ctx) 14 | logger-config (.getLoggerConfig config logger-name)] 15 | (.setLevel logger-config level) 16 | (.updateLoggers ctx))) 17 | 18 | ;; Here is an example of using it to change the root logger to "DEBUG" and the 19 | ;; back to "INFO" after a request has been completed 20 | (defn post-page-with-debug [] 21 | (change-log-level! LogManager/ROOT_LOGGER_NAME Level/DEBUG) 22 | (http/post "https://httpbin.org/post" {:body "this is a test"}) 23 | (change-log-level! LogManager/ROOT_LOGGER_NAME Level/INFO)) 24 | -------------------------------------------------------------------------------- /examples/progress_download.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.examples.progress-download 2 | (:require [clj-http.client :as http] 3 | [clojure.java.io :refer [output-stream]]) 4 | (:import (org.apache.commons.io.input CountingInputStream))) 5 | 6 | (defn print-progress-bar 7 | "Render a simple progress bar given the progress and total. If the total is zero 8 | the progress will run as indeterminated." 9 | ([progress total] (print-progress-bar progress total {})) 10 | ([progress total {:keys [bar-width] 11 | :or {bar-width 50}}] 12 | (if (pos? total) 13 | (let [pct (/ progress total) 14 | render-bar (fn [] 15 | (let [bars (Math/floor (* pct bar-width)) 16 | pad (- bar-width bars)] 17 | (str (clojure.string/join (repeat bars "=")) 18 | (clojure.string/join (repeat pad " ")))))] 19 | (print (str "[" (render-bar) "] " 20 | (int (* pct 100)) "% " 21 | progress "/" total))) 22 | (let [render-bar (fn [] (clojure.string/join (repeat bar-width "-")))] 23 | (print (str "[" (render-bar) "] " 24 | progress "/?")))))) 25 | 26 | (defn insert-at [v idx val] 27 | "Addes value into a vector at an specific index." 28 | (-> (subvec v 0 idx) 29 | (conj val) 30 | (into (subvec v idx)))) 31 | 32 | (defn insert-after [v needle val] 33 | "Finds an item into a vector and adds val just after it. 34 | If needle is not found, the input vector will be returned." 35 | (let [index (.indexOf v needle)] 36 | (if (neg? index) 37 | v 38 | (insert-at v (inc index) val)))) 39 | 40 | (defn wrap-downloaded-bytes-counter 41 | "Middleware that provides an CountingInputStream wrapping the stream output" 42 | [client] 43 | (fn [req] 44 | (let [resp (client req) 45 | counter (CountingInputStream. (:body resp))] 46 | (merge resp {:body counter 47 | :downloaded-bytes-counter counter})))) 48 | 49 | (defn download-with-progress [url target] 50 | (http/with-middleware 51 | (-> http/default-middleware 52 | (insert-after http/wrap-redirects wrap-downloaded-bytes-counter) 53 | (conj http/wrap-lower-case-headers)) 54 | (let [request (http/get url {:as :stream}) 55 | length (Integer. (get-in request [:headers "content-length"] 0)) 56 | buffer-size (* 1024 10)] 57 | (println) 58 | (with-open [input (:body request) 59 | output (output-stream target)] 60 | (let [buffer (make-array Byte/TYPE buffer-size) 61 | counter (:downloaded-bytes-counter request)] 62 | (loop [] 63 | (let [size (.read input buffer)] 64 | (when (pos? size) 65 | (.write output buffer 0 size) 66 | (print "\r") 67 | (print-progress-bar (.getByteCount counter) length) 68 | (recur)))))) 69 | (println)))) 70 | 71 | ;; Example of progress bar output (sample steps) 72 | ;; 73 | ;; [=== ] 7% 2094930/26572400 74 | ;; [============================== ] 60% 16062930/26572400 75 | ;; [========================================= ] 83% 22290930/26572400 76 | ;; [==================================================] 100% 26572400/26572400 77 | ;; 78 | ;; In case the content-length is unknown, the bar will be displayed as: 79 | ;; 80 | ;; [--------------------------------------------------] 4211440/? 81 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject clj-http "3.13.1-SNAPSHOT" 2 | :description "A Clojure HTTP library wrapping the Apache HttpComponents client." 3 | :url "https://github.com/dakrone/clj-http/" 4 | :license {:name "The MIT License" 5 | :url "http://opensource.org/licenses/mit-license.php" 6 | :distribution :repo} 7 | :global-vars {*warn-on-reflection* false} 8 | :min-lein-version "2.0.0" 9 | :exclusions [org.clojure/clojure] 10 | :dependencies [[org.apache.httpcomponents/httpcore "4.4.16"] 11 | [org.apache.httpcomponents/httpclient "4.5.14"] 12 | [org.apache.httpcomponents/httpclient-cache "4.5.14"] 13 | [org.apache.httpcomponents/httpasyncclient "4.1.5"] 14 | [org.apache.httpcomponents/httpmime "4.5.14"] 15 | [commons-codec "1.16.1"] 16 | [commons-io "2.16.1"] 17 | [slingshot "0.12.2"] 18 | [potemkin "0.4.7"]] 19 | :resource-paths ["resources"] 20 | :profiles {:dev {:dependencies [;; optional deps 21 | [cheshire "5.13.0"] 22 | [crouton "0.1.2" :exclusions [[org.jsoup/jsoup]]] 23 | [org.jsoup/jsoup "1.17.2"] 24 | [org.clojure/tools.reader "1.4.1"] 25 | [com.cognitect/transit-clj "1.0.333"] 26 | [ring/ring-codec "1.2.0"] 27 | ;; other (testing) deps 28 | [org.clojure/clojure "1.11.2"] 29 | [org.clojure/tools.logging "1.3.0"] 30 | [ring/ring-jetty-adapter "1.12.1"] 31 | [ring/ring-devel "1.12.1"] 32 | [javax.servlet/javax.servlet-api "4.0.1"] 33 | ;; caching example deps 34 | [org.clojure/core.cache "1.1.234"] 35 | ;; logging 36 | [org.apache.logging.log4j/log4j-api "2.23.1"] 37 | [org.apache.logging.log4j/log4j-core "2.23.1"] 38 | [org.apache.logging.log4j/log4j-1.2-api "2.23.1"]] 39 | :plugins [[lein-ancient "0.7.0"] 40 | [jonase/eastwood "0.2.5"] 41 | [lein-kibit "0.1.5"] 42 | [lein-nvd "0.5.2"]]} 43 | :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]} 44 | :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]} 45 | :1.10 {:dependencies [[org.clojure/clojure "1.10.1"]]}} 46 | :aliases {"all" ["with-profile" "dev,1.8:dev,1.9:dev,1.10:dev"]} 47 | :plugins [[codox "0.6.4"]] 48 | :test-selectors {:default #(not (:integration %)) 49 | :integration :integration 50 | :all (constantly true)}) 51 | -------------------------------------------------------------------------------- /resources/example-log4j2.properties: -------------------------------------------------------------------------------- 1 | ### 2 | # While no means required, this is an example log4j2.properties that you can use 3 | # for debugging clj-http (mostly the apache http client side). See the readme or 4 | # examples directory for how to use it. 5 | 6 | # Change this to "debug" to get debugging information 7 | rootLogger.level = info 8 | rootLogger.appenderRef.console.ref = console 9 | rootLogger.appenderRef.rolling.ref = fileLogger 10 | 11 | # Give directory path where log files should get stored 12 | property.basePath = ./log/ 13 | status = error 14 | 15 | # ConsoleAppender will print logs on console 16 | appender.console.type = Console 17 | appender.console.name = console 18 | appender.console.layout.type = PatternLayout 19 | # Specify the pattern of the logs 20 | appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n 21 | 22 | # RollingFileAppender will print logs in file which can be rotated based on time 23 | # or size 24 | appender.rolling.type = RollingFile 25 | appender.rolling.name = fileLogger 26 | appender.rolling.fileName=${basePath}/clj-http.log 27 | appender.rolling.filePattern=${basePath}clj-http_%d{yyyyMMdd}.log.gz 28 | appender.rolling.layout.type = PatternLayout 29 | appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n 30 | appender.rolling.policies.type = Policies 31 | 32 | # Mention package name here in place of example. Classes in this package or 33 | # subpackages will use ConsoleAppender and RollingFileAppender for logging 34 | logger.example.name = example 35 | logger.example.level = debug 36 | logger.example.additivity = false 37 | logger.example.appenderRef.rolling.ref = fileLogger 38 | logger.example.appenderRef.console.ref = console 39 | -------------------------------------------------------------------------------- /src/clj_http/conn_mgr.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.conn-mgr 2 | "Utility methods for Scheme registries and HTTP connection managers" 3 | (:require [clj-http.util :refer [opt]] 4 | [clojure.java.io :as io]) 5 | (:import [java.net InetSocketAddress Proxy Proxy$Type Socket] 6 | java.security.KeyStore 7 | [javax.net.ssl HostnameVerifier KeyManager SSLContext TrustManager] 8 | [org.apache.http.config ConnectionConfig Registry RegistryBuilder SocketConfig] 9 | org.apache.http.conn.HttpClientConnectionManager 10 | org.apache.http.conn.socket.PlainConnectionSocketFactory 11 | [org.apache.http.conn.ssl DefaultHostnameVerifier NoopHostnameVerifier SSLConnectionSocketFactory SSLContexts TrustStrategy] 12 | [org.apache.http.impl.conn BasicHttpClientConnectionManager PoolingHttpClientConnectionManager] 13 | org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager 14 | org.apache.http.impl.nio.DefaultHttpClientIODispatch 15 | [org.apache.http.impl.nio.reactor DefaultConnectingIOReactor IOReactorConfig] 16 | [org.apache.http.nio.conn NHttpClientConnectionManager NoopIOSessionStrategy] 17 | org.apache.http.nio.conn.ssl.SSLIOSessionStrategy 18 | org.apache.http.nio.protocol.HttpAsyncRequestExecutor)) 19 | 20 | ;; -- Interop Helpers --------------------------------------------------------- 21 | (defn ^Registry into-registry [registry] 22 | (cond 23 | (instance? Registry registry) 24 | registry 25 | 26 | (map? registry) 27 | (let [registry-builder (RegistryBuilder/create)] 28 | (doseq [[k v] registry] 29 | (.register registry-builder k v)) 30 | (.build registry-builder)) 31 | 32 | :else 33 | (throw (IllegalArgumentException. "Cannot coerce into a Registry")))) 34 | 35 | ;; -- SocketFactory ----------------------------------------------------------- 36 | (defn ^SSLConnectionSocketFactory SSLGenericSocketFactory 37 | "Given a function that returns a new socket, create an 38 | SSLConnectionSocketFactory that will use that socket." 39 | ([socket-factory] 40 | (SSLGenericSocketFactory socket-factory nil)) 41 | ([socket-factory ^SSLContext ssl-context] 42 | (let [^SSLContext ssl-context' (or ssl-context (SSLContexts/createDefault))] 43 | (proxy [SSLConnectionSocketFactory] [ssl-context'] 44 | (connectSocket [timeout socket host remoteAddress localAddress context] 45 | (let [^SSLConnectionSocketFactory this this] ;; avoid reflection 46 | (proxy-super connectSocket timeout (socket-factory) host remoteAddress 47 | localAddress context))))))) 48 | 49 | (defn ^PlainConnectionSocketFactory PlainGenericSocketFactory 50 | "Given a Function that returns a new socket, create a 51 | PlainConnectionSocketFactory that will use that socket." 52 | [socket-factory] 53 | (proxy [PlainConnectionSocketFactory] [] 54 | (createSocket [context] 55 | (socket-factory)))) 56 | 57 | (defn socks-proxied-socket 58 | "Create a Socket proxied through socks, using the given hostname and port" 59 | [^String hostname ^Integer port] 60 | (Socket. (Proxy. Proxy$Type/SOCKS (InetSocketAddress. hostname port)))) 61 | 62 | ;; -- SSL Contexts ------------------------------------------------------------ 63 | (defn ^KeyStore get-keystore* 64 | [keystore-file keystore-type ^String keystore-pass] 65 | (when keystore-file 66 | (let [keystore (KeyStore/getInstance (or keystore-type 67 | (KeyStore/getDefaultType)))] 68 | (with-open [is (io/input-stream keystore-file)] 69 | (.load keystore is (when keystore-pass (.toCharArray keystore-pass))) 70 | keystore)))) 71 | 72 | (defn ^KeyStore get-keystore [keystore & args] 73 | (if (instance? KeyStore keystore) 74 | keystore 75 | (apply get-keystore* keystore args))) 76 | 77 | (defn- ssl-context-for-keystore 78 | ;; TODO: use something else for passwords 79 | ;; Note: JVM strings aren't ideal for passwords - see 80 | ;; https://tinyurl.com/azm3ab9 81 | [{:keys [keystore keystore-type ^String keystore-pass 82 | trust-store trust-store-type trust-store-pass]}] 83 | (let [ks (get-keystore keystore keystore-type keystore-pass) 84 | ts (get-keystore trust-store trust-store-type trust-store-pass)] 85 | (-> (SSLContexts/custom) 86 | (.loadKeyMaterial 87 | ks (when keystore-pass 88 | (.toCharArray keystore-pass))) 89 | (.loadTrustMaterial 90 | ts nil) 91 | (.build)))) 92 | 93 | (defn- ssl-context-for-trust-or-key-manager 94 | "Given an instance or seqable data structure of TrustManager or KeyManager 95 | will create and return an SSLContexts object including the resulting managers" 96 | [{:keys [trust-managers key-managers]}] 97 | (let [x-or-xs->x-array (fn [type x-or-xs] 98 | (cond 99 | (or (-> x-or-xs class .isArray) 100 | (sequential? x-or-xs)) 101 | (into-array type (seq x-or-xs)) 102 | 103 | :else 104 | (into-array type [x-or-xs]))) 105 | trust-managers (when trust-managers 106 | (x-or-xs->x-array TrustManager trust-managers)) 107 | key-managers (when key-managers 108 | (x-or-xs->x-array KeyManager key-managers))] 109 | (doto (.build (SSLContexts/custom)) 110 | (.init key-managers trust-managers nil)))) 111 | 112 | (defn- ssl-context-insecure 113 | "Creates a SSL Context that trusts all material." 114 | [] 115 | (-> (SSLContexts/custom) 116 | (.loadTrustMaterial nil (reify TrustStrategy 117 | (isTrusted [_ chain auth-type] true))) 118 | (.build))) 119 | 120 | (defn ^SSLContext get-ssl-context 121 | "Gets the SSL Context from a request or connection pool settings" 122 | [{:keys [keystore trust-store key-managers trust-managers] :as config}] 123 | (cond (or keystore trust-store) 124 | (ssl-context-for-keystore config) 125 | 126 | (or key-managers trust-managers) 127 | (ssl-context-for-trust-or-key-manager config) 128 | 129 | (opt config :insecure) 130 | (ssl-context-insecure) 131 | 132 | :else 133 | (SSLContexts/createDefault))) 134 | 135 | (defn ^HostnameVerifier get-hostname-verifier [config] 136 | (if (opt config :insecure) 137 | NoopHostnameVerifier/INSTANCE 138 | (DefaultHostnameVerifier.))) 139 | 140 | ;; -- Connection Managers ----------------------------------------------------- 141 | (defn make-socks-proxied-conn-manager 142 | "Given an optional hostname and a port, create a connection manager that's 143 | proxied using a SOCKS proxy." 144 | ([^String hostname ^Integer port] 145 | (make-socks-proxied-conn-manager hostname port {})) 146 | ([^String hostname ^Integer port 147 | {:keys [keystore keystore-type keystore-pass 148 | trust-store trust-store-type trust-store-pass 149 | trust-managers key-managers] :as config}] 150 | (let [socket-factory #(socks-proxied-socket hostname port) 151 | registry (into-registry 152 | {"http" (PlainGenericSocketFactory socket-factory) 153 | "https" (SSLGenericSocketFactory socket-factory (get-ssl-context config))})] 154 | (PoolingHttpClientConnectionManager. registry)))) 155 | 156 | (defn ^BasicHttpClientConnectionManager make-regular-conn-manager 157 | [{:keys [dns-resolver 158 | keystore trust-store 159 | key-managers trust-managers 160 | socket-timeout] :as config}] 161 | 162 | (let [registry (into-registry 163 | {"http" (PlainConnectionSocketFactory/getSocketFactory) 164 | "https" (SSLConnectionSocketFactory. 165 | (get-ssl-context config) 166 | (get-hostname-verifier config))}) 167 | conn-manager (BasicHttpClientConnectionManager. registry 168 | nil nil 169 | dns-resolver)] 170 | (when socket-timeout 171 | (.setSocketConfig conn-manager 172 | (-> (.getSocketConfig conn-manager) 173 | (SocketConfig/copy) 174 | (.setSoTimeout socket-timeout) ;modify only the socket-timeout 175 | (.build)))) 176 | conn-manager)) 177 | 178 | (defn- ^DefaultConnectingIOReactor make-ioreactor 179 | [{:keys [connect-timeout interest-op-queued io-thread-count rcv-buf-size 180 | select-interval shutdown-grace-period snd-buf-size 181 | so-keep-alive so-linger so-timeout tcp-no-delay]}] 182 | (as-> (IOReactorConfig/custom) c 183 | (if-some [v connect-timeout] (.setConnectTimeout c v) c) 184 | (if-some [v interest-op-queued] (.setInterestOpQueued c v) c) 185 | (if-some [v io-thread-count] (.setIoThreadCount c v) c) 186 | (if-some [v rcv-buf-size] (.setRcvBufSize c v) c) 187 | (if-some [v select-interval] (.setSelectInterval c v) c) 188 | (if-some [v shutdown-grace-period] (.setShutdownGracePeriod c v) c) 189 | (if-some [v snd-buf-size] (.setSndBufSize c v) c) 190 | (if-some [v so-keep-alive] (.setSoKeepAlive c v) c) 191 | (if-some [v so-linger] (.setSoLinger c v) c) 192 | (if-some [v so-timeout] (.setSoTimeout c v) c) 193 | (if-some [v tcp-no-delay] (.setTcpNoDelay c v) c) 194 | (DefaultConnectingIOReactor. (.build c)))) 195 | 196 | (defn ^PoolingNHttpClientConnectionManager 197 | make-regular-async-conn-manager 198 | [{:keys [keystore trust-store 199 | key-managers trust-managers] :as config}] 200 | (let [^Registry registry (into-registry 201 | {"http" (NoopIOSessionStrategy/INSTANCE) 202 | "https" (SSLIOSessionStrategy. 203 | (get-ssl-context config) 204 | (get-hostname-verifier config))}) 205 | io-reactor (make-ioreactor {:shutdown-grace-period 1})] 206 | (doto (PoolingNHttpClientConnectionManager. io-reactor registry) 207 | (.setMaxTotal 1)))) 208 | 209 | (definterface ReuseableAsyncConnectionManager) 210 | 211 | ;; need the fully qualified class name because this fn is later used in a 212 | ;; macro from a different ns 213 | (defn ^org.apache.http.impl.conn.PoolingHttpClientConnectionManager 214 | make-reusable-conn-manager* 215 | "Given an timeout and optional insecure? flag, create a 216 | PoolingHttpClientConnectionManager with seconds set as the 217 | timeout value." 218 | [{:keys [dns-resolver 219 | timeout 220 | keystore trust-store 221 | key-managers trust-managers] :as config}] 222 | (let [registry (into-registry 223 | {"http" (PlainConnectionSocketFactory/getSocketFactory) 224 | "https" (SSLConnectionSocketFactory. 225 | (get-ssl-context config) 226 | (get-hostname-verifier config))})] 227 | (PoolingHttpClientConnectionManager. 228 | registry nil nil dns-resolver timeout java.util.concurrent.TimeUnit/SECONDS))) 229 | 230 | (defn reusable? [conn-mgr] 231 | (or (instance? PoolingHttpClientConnectionManager conn-mgr) 232 | (instance? ReuseableAsyncConnectionManager conn-mgr))) 233 | 234 | (defn ^PoolingHttpClientConnectionManager make-reusable-conn-manager 235 | "Creates a default pooling connection manager with the specified options. 236 | 237 | The following options are supported: 238 | 239 | :timeout - Time that connections are left open before automatically closing 240 | default: 5 241 | :threads - Maximum number of threads that will be used for connecting 242 | default: 4 243 | :default-per-route - Maximum number of simultaneous connections per host 244 | default: 2 245 | :insecure? - Boolean flag to specify allowing insecure HTTPS connections 246 | default: false 247 | 248 | :keystore - keystore file to be used for connection manager 249 | :keystore-pass - keystore password 250 | :trust-store - trust store file to be used for connection manager 251 | :trust-store-pass - trust store password 252 | 253 | :key-managers - KeyManager objects to be used for connection manager 254 | :trust-managers - TrustManager objects to be used for connection manager 255 | 256 | :dns-resolver - Use a custom DNS resolver instead of the default DNS resolver. 257 | 258 | Note that :insecure? and :keystore/:trust-store/:key-managers/:trust-managers options are mutually exclusive 259 | 260 | Note that :key-managers/:trust-managers have precedence over :keystore/:trust-store options 261 | 262 | 263 | If the value 'nil' is specified or the value is not set, the default value 264 | will be used." 265 | [opts] 266 | (let [timeout (or (:timeout opts) 5) 267 | threads (or (:threads opts) 4) 268 | default-per-route (:default-per-route opts) 269 | insecure? (opt opts :insecure) 270 | leftovers (dissoc opts :timeout :threads :insecure? :insecure) 271 | conn-man (make-reusable-conn-manager* (merge {:timeout timeout 272 | :insecure? insecure?} 273 | leftovers))] 274 | (.setMaxTotal conn-man threads) 275 | (when default-per-route 276 | (.setDefaultMaxPerRoute conn-man default-per-route)) 277 | conn-man)) 278 | 279 | (defn- ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager* 280 | [{:keys [dns-resolver 281 | timeout keystore trust-store io-config 282 | key-managers trust-managers] :as config}] 283 | (let [registry (into-registry 284 | {"http" (NoopIOSessionStrategy/INSTANCE) 285 | "https" (SSLIOSessionStrategy. 286 | (get-ssl-context config) 287 | (get-hostname-verifier config))}) 288 | io-reactor (make-ioreactor io-config) 289 | protocol-handler (HttpAsyncRequestExecutor.) 290 | io-event-dispatch (DefaultHttpClientIODispatch. protocol-handler 291 | ConnectionConfig/DEFAULT)] 292 | (future (.execute io-reactor io-event-dispatch)) 293 | (proxy [PoolingNHttpClientConnectionManager ReuseableAsyncConnectionManager] 294 | [io-reactor nil registry nil dns-resolver timeout 295 | java.util.concurrent.TimeUnit/SECONDS]))) 296 | 297 | (defn ^PoolingNHttpClientConnectionManager make-reusable-async-conn-manager 298 | "Creates a default pooling async connection manager with the specified 299 | options. Handles the same options as make-reusable-conn-manager plus 300 | :io-config which should be a map containing some of the following keys: 301 | 302 | :connect-timeout - int the default connect timeout value for connection 303 | requests (default 0, meaning no timeout) 304 | :interest-op-queued - boolean, whether or not I/O interest operations are to 305 | be queued and executed asynchronously or to be applied to the underlying 306 | SelectionKey immediately (default false) 307 | :io-thread-count - int, the number of I/O dispatch threads to be used 308 | (default is the number of available processors) 309 | :rcv-buf-size - int the default value of the SO_RCVBUF parameter for 310 | newly created sockets (default is 0, meaning the system default) 311 | :select-interval - long, time interval in milliseconds at which to check for 312 | timed out sessions and session requests (default 1000) 313 | :shutdown-grace-period - long, grace period in milliseconds to wait for 314 | individual worker threads to terminate cleanly (default 500) 315 | :snd-buf-size - int, the default value of the SO_SNDBUF parameter for 316 | newly created sockets (default is 0, meaning the system default) 317 | :so-keep-alive - boolean, the default value of the SO_KEEPALIVE parameter for 318 | newly created sockets (default false) 319 | :so-linger - int, the default value of the SO_LINGER parameter for 320 | newly created sockets (default -1) 321 | :so-timeout - int, the default socket timeout value for I/O operations 322 | (default 0, meaning no timeout) 323 | :tcp-no-delay - boolean, the default value of the TCP_NODELAY parameter for 324 | newly created sockets (default true) 325 | 326 | If the value 'nil' is specified or the value is not set, the default value 327 | will be used." 328 | [opts] 329 | (let [timeout (or (:timeout opts) 5) 330 | threads (or (:threads opts) 4) 331 | default-per-route (:default-per-route opts) 332 | insecure? (opt opts :insecure) 333 | leftovers (dissoc opts :timeout :threads :insecure? :insecure) 334 | conn-man (make-reusable-async-conn-manager* 335 | (merge {:timeout timeout :insecure? insecure?} leftovers))] 336 | (.setMaxTotal conn-man threads) 337 | (when default-per-route 338 | (.setDefaultMaxPerRoute conn-man default-per-route)) 339 | conn-man)) 340 | 341 | (defn ^PoolingNHttpClientConnectionManager make-reuseable-async-conn-manager 342 | "Wraps correctly-spelled version - keeping for backwards compatibility." 343 | [opts] 344 | (make-reusable-async-conn-manager opts)) 345 | 346 | (defmulti shutdown-manager 347 | "Shut down the given connection manager, if it is not nil" 348 | class) 349 | (defmethod shutdown-manager nil [conn-mgr] nil) 350 | (defmethod shutdown-manager org.apache.http.conn.HttpClientConnectionManager 351 | [^HttpClientConnectionManager conn-mgr] (.shutdown conn-mgr)) 352 | (defmethod shutdown-manager 353 | org.apache.http.nio.conn.NHttpClientConnectionManager 354 | [^NHttpClientConnectionManager conn-mgr] (.shutdown conn-mgr)) 355 | 356 | (def ^:dynamic *connection-manager* 357 | "connection manager to be rebound during request execution" 358 | nil) 359 | 360 | (def ^:dynamic *async-connection-manager* 361 | "connection manager to be rebound during async request execution" 362 | nil) 363 | -------------------------------------------------------------------------------- /src/clj_http/cookies.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.cookies 2 | "Namespace dealing with HTTP cookies" 3 | (:require [clj-http.util :refer [opt]] 4 | [clojure.string :refer [blank? join lower-case]]) 5 | (:import org.apache.http.client.CookieStore 6 | [org.apache.http.cookie ClientCookie CookieOrigin CookieSpec] 7 | org.apache.http.Header 8 | org.apache.http.impl.client.BasicCookieStore 9 | [org.apache.http.impl.cookie BasicClientCookie2 BrowserCompatSpecFactory] 10 | org.apache.http.message.BasicHeader 11 | org.apache.http.protocol.BasicHttpContext)) 12 | 13 | (defn cookie-spec ^org.apache.http.cookie.CookieSpec [] 14 | (.create 15 | (BrowserCompatSpecFactory.) 16 | (BasicHttpContext.))) 17 | 18 | (defn compact-map 19 | "Removes all map entries where value is nil." 20 | [m] 21 | (reduce (fn [newm k] 22 | (if (not (nil? (get m k))) 23 | (assoc newm k (get m k)) 24 | newm)) 25 | {} (keys m))) 26 | 27 | (defn to-cookie 28 | "Converts a ClientCookie object into a tuple where the first item is 29 | the name of the cookie and the second item the content of the 30 | cookie." 31 | [^ClientCookie cookie] 32 | [(.getName cookie) 33 | (compact-map 34 | {:comment (.getComment cookie) 35 | :comment-url (.getCommentURL cookie) 36 | :discard (not (.isPersistent cookie)) 37 | :domain (.getDomain cookie) 38 | :expires (when (.getExpiryDate cookie) (.getExpiryDate cookie)) 39 | :path (.getPath cookie) 40 | :ports (when (.getPorts cookie) (seq (.getPorts cookie))) 41 | :secure (.isSecure cookie) 42 | :value (.getValue cookie) 43 | :version (.getVersion cookie)})]) 44 | 45 | (defn ^BasicClientCookie2 46 | to-basic-client-cookie 47 | "Converts a cookie seq into a BasicClientCookie2." 48 | [[cookie-name cookie-content]] 49 | (doto (BasicClientCookie2. (name cookie-name) 50 | (name (:value cookie-content))) 51 | (.setComment (:comment cookie-content)) 52 | (.setCommentURL (:comment-url cookie-content)) 53 | (.setDiscard (:discard cookie-content true)) 54 | (.setDomain (:domain cookie-content)) 55 | (.setExpiryDate (:expires cookie-content)) 56 | (.setPath (:path cookie-content)) 57 | (.setPorts (int-array (:ports cookie-content))) 58 | (.setSecure (:secure cookie-content false)) 59 | (.setVersion (:version cookie-content 0)))) 60 | 61 | (defn decode-cookie 62 | "Decode the Set-Cookie string into a cookie seq." 63 | [set-cookie-str] 64 | (if-not (blank? set-cookie-str) 65 | ;; I just want to parse a cookie without providing origin. How? 66 | (let [domain (lower-case (str (gensym))) 67 | origin (CookieOrigin. domain 80 "/" false) 68 | [cookie-name cookie-content] (-> (cookie-spec) 69 | (.parse (BasicHeader. 70 | "set-cookie" 71 | set-cookie-str) 72 | origin) 73 | first 74 | to-cookie)] 75 | [cookie-name 76 | (if (= domain (:domain cookie-content)) 77 | (dissoc cookie-content :domain) cookie-content)]))) 78 | 79 | (defn decode-cookies 80 | "Converts a cookie string or seq of strings into a cookie map." 81 | [cookies] 82 | (reduce #(assoc %1 (first %2) (second %2)) {} 83 | (map decode-cookie (if (sequential? cookies) cookies [cookies])))) 84 | 85 | (defn decode-cookie-header 86 | "Decode the Set-Cookie header into the cookies key." 87 | [response] 88 | (if-let [cookies (get (:headers response) "set-cookie")] 89 | (assoc response 90 | :cookies (decode-cookies cookies) 91 | :headers (dissoc (:headers response) "set-cookie")) 92 | response)) 93 | 94 | (defn encode-cookie 95 | "Encode the cookie into a string used by the Cookie header." 96 | [cookie] 97 | (when-let [header (-> (cookie-spec) 98 | (.formatCookies [(to-basic-client-cookie cookie)]) 99 | first)] 100 | (.getValue ^Header header))) 101 | 102 | (defn encode-cookies 103 | "Encode the cookie map into a string." 104 | [cookie-map] (join ";" (map encode-cookie (seq cookie-map)))) 105 | 106 | (defn encode-cookie-header 107 | "Encode the :cookies key of the request into a Cookie header." 108 | [request] 109 | (if (:cookies request) 110 | (-> request 111 | (assoc-in [:headers "Cookie"] (encode-cookies (:cookies request))) 112 | (dissoc :cookies)) 113 | request)) 114 | 115 | (defn- cookies-response 116 | [request response] 117 | (if (= false (opt request :decode-cookies)) 118 | response 119 | (decode-cookie-header response))) 120 | 121 | (defn wrap-cookies 122 | "Middleware wrapping cookie handling. Handles converting 123 | the :cookies request parameter into the 'Cookies' header for an HTTP 124 | request." 125 | [client] 126 | (fn 127 | ([request] 128 | (cookies-response request (client (encode-cookie-header request)))) 129 | ([request respond raise] 130 | (client (encode-cookie-header request) 131 | #(respond (cookies-response request %)) 132 | raise)))) 133 | 134 | (defn cookie-store 135 | "Returns a new, empty instance of the default implementation of the 136 | org.apache.http.client.CookieStore interface." 137 | [] 138 | (BasicCookieStore.)) 139 | 140 | (defn get-cookies 141 | "Given a cookie-store, return a map of cookie name to a map of cookie values." 142 | [^CookieStore cookie-store] 143 | (when cookie-store 144 | (into {} (map to-cookie (.getCookies cookie-store))))) 145 | 146 | (defn add-cookie 147 | "Add a ClientCookie to a cookie-store" 148 | [^CookieStore cookie-store ^ClientCookie cookie] 149 | (.addCookie cookie-store cookie)) 150 | 151 | (defn clear-cookies 152 | "Clears all cookies from cookie-store" 153 | [^CookieStore cookie-store] 154 | (.clear cookie-store)) 155 | -------------------------------------------------------------------------------- /src/clj_http/core.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.core 2 | "Core HTTP request/response implementation. Rewrite for Apache 4.3" 3 | (:require [clj-http.conn-mgr :as conn] 4 | [clj-http.headers :as headers] 5 | [clj-http.multipart :as mp] 6 | [clj-http.util :refer [opt]] 7 | clojure.pprint) 8 | (:import [java.io ByteArrayOutputStream FilterInputStream InputStream] 9 | [java.net InetAddress ProxySelector URI URL] 10 | java.util.Locale 11 | [org.apache.http HeaderIterator HttpEntity HttpEntityEnclosingRequest HttpHost HttpRequestInterceptor HttpResponse HttpResponseInterceptor ProtocolException] 12 | [org.apache.http.auth AuthScope NTCredentials UsernamePasswordCredentials] 13 | [org.apache.http.client CredentialsProvider HttpRequestRetryHandler RedirectStrategy] 14 | org.apache.http.client.cache.HttpCacheContext 15 | [org.apache.http.client.config CookieSpecs RequestConfig] 16 | [org.apache.http.client.methods CloseableHttpResponse HttpDelete HttpEntityEnclosingRequestBase HttpGet HttpHead HttpOptions HttpPatch HttpPost HttpPut HttpRequestBase HttpUriRequest] 17 | org.apache.http.client.protocol.HttpClientContext 18 | org.apache.http.client.utils.URIUtils 19 | org.apache.http.config.RegistryBuilder 20 | org.apache.http.conn.routing.HttpRoutePlanner 21 | org.apache.http.cookie.CookieSpecProvider 22 | [org.apache.http.entity ByteArrayEntity StringEntity] 23 | [org.apache.http.impl.client BasicCredentialsProvider CloseableHttpClient DefaultRedirectStrategy HttpClientBuilder HttpClients LaxRedirectStrategy] 24 | [org.apache.http.impl.client.cache CacheConfig CachingHttpClientBuilder] 25 | [org.apache.http.impl.conn DefaultProxyRoutePlanner SystemDefaultRoutePlanner] 26 | [org.apache.http.impl.nio.client CloseableHttpAsyncClient HttpAsyncClientBuilder HttpAsyncClients])) 27 | 28 | (def CUSTOM_COOKIE_POLICY "_custom") 29 | 30 | (defn parse-headers 31 | "Takes a HeaderIterator and returns a map of names to values. 32 | 33 | If a name appears more than once (like `set-cookie`) then the value 34 | will be a vector containing the values in the order they appeared 35 | in the headers." 36 | [^HeaderIterator headers & [use-header-maps-in-response?]] 37 | (if-not use-header-maps-in-response? 38 | (->> (headers/header-iterator-seq headers) 39 | (map (fn [[k v]] 40 | [(.toLowerCase ^String k) v])) 41 | (reduce (fn [hs [k v]] 42 | (headers/assoc-join hs k v)) 43 | {})) 44 | (->> (headers/header-iterator-seq headers) 45 | (reduce (fn [hs [k v]] 46 | (headers/assoc-join hs k v)) 47 | (headers/header-map))))) 48 | 49 | (defn graceful-redirect-strategy 50 | "Similar to the default redirect strategy, however, does not throw an error 51 | when the maximum number of redirects has been reached. Still supports 52 | validating that the new redirect host is not empty." 53 | [req] 54 | (let [validate? (opt req :validate-redirects)] 55 | (reify RedirectStrategy 56 | (getRedirect [this request response context] 57 | (let [new-request (.getRedirect DefaultRedirectStrategy/INSTANCE 58 | request response context)] 59 | (when (or validate? (nil? validate?)) 60 | (let [uri (.getURI new-request) 61 | new-host (URIUtils/extractHost uri)] 62 | (when (nil? new-host) 63 | (throw 64 | (ProtocolException. 65 | (str "Redirect URI does not specify a valid host name: " 66 | uri)))))) 67 | new-request)) 68 | 69 | (isRedirected [this request response context] 70 | (let [^HttpClientContext typed-context context 71 | max-redirects (-> (.getRequestConfig typed-context) 72 | .getMaxRedirects) 73 | num-redirects (count (.getRedirectLocations typed-context))] 74 | (if (<= max-redirects num-redirects) 75 | false 76 | (.isRedirected DefaultRedirectStrategy/INSTANCE 77 | request response typed-context))))))) 78 | 79 | (defn default-redirect-strategy 80 | "Proxies calls to whatever original redirect strategy is passed in, however, 81 | if :validate-redirects is set in the request, checks that the redirected host 82 | is not empty." 83 | [^RedirectStrategy original req] 84 | (let [validate? (opt req :validate-redirects)] 85 | (reify RedirectStrategy 86 | (getRedirect [this request response context] 87 | (let [new-request (.getRedirect original request response context)] 88 | (when (or validate? (nil? validate?)) 89 | (let [uri (.getURI new-request) 90 | new-host (URIUtils/extractHost uri)] 91 | (when (nil? new-host) 92 | (throw 93 | (ProtocolException. 94 | (str "Redirect URI does not specify a valid host name: " 95 | uri)))))) 96 | new-request)) 97 | 98 | (isRedirected [this request response context] 99 | (.isRedirected original request response context))))) 100 | 101 | (defn get-redirect-strategy [{:keys [redirect-strategy] :as req}] 102 | (case redirect-strategy 103 | :none (reify RedirectStrategy 104 | (getRedirect [this request response context] nil) 105 | (isRedirected [this request response context] false)) 106 | 107 | ;; Like default, but does not throw exceptions when max redirects is 108 | ;; reached. 109 | :graceful (graceful-redirect-strategy req) 110 | 111 | :default (default-redirect-strategy DefaultRedirectStrategy/INSTANCE req) 112 | :lax (default-redirect-strategy (LaxRedirectStrategy.) req) 113 | nil (default-redirect-strategy DefaultRedirectStrategy/INSTANCE req) 114 | 115 | ;; use directly as reifed RedirectStrategy 116 | redirect-strategy)) 117 | 118 | (defn ^HttpClientBuilder add-retry-handler [^HttpClientBuilder builder handler] 119 | (when handler 120 | (.setRetryHandler 121 | builder 122 | (proxy [HttpRequestRetryHandler] [] 123 | (retryRequest [e cnt context] 124 | (handler e cnt context))))) 125 | builder) 126 | 127 | (defn create-custom-cookie-policy-registry 128 | "Given a function that will take an HttpContext and return a CookieSpec, 129 | create a new Registry for the cookie policy under the CUSTOM_COOKIE_POLICY 130 | string." 131 | [cookie-spec-fn] 132 | (-> (RegistryBuilder/create) 133 | (.register CUSTOM_COOKIE_POLICY 134 | (proxy [CookieSpecProvider] [] 135 | (create [context] 136 | (cookie-spec-fn context)))) 137 | (.build))) 138 | 139 | (defmulti get-cookie-policy 140 | "Method to retrieve the cookie policy that should be used for the request. 141 | This is a multimethod that may be extended to return your own cookie policy. 142 | Dispatches based on the `:cookie-policy` key in the request map." 143 | (fn get-cookie-dispatch [request] (:cookie-policy request))) 144 | 145 | (defmethod get-cookie-policy :none none-cookie-policy 146 | [_] CookieSpecs/IGNORE_COOKIES) 147 | (defmethod get-cookie-policy :default default-cookie-policy 148 | [_] CookieSpecs/DEFAULT) 149 | (defmethod get-cookie-policy nil nil-cookie-policy 150 | [_] CookieSpecs/DEFAULT) 151 | (defmethod get-cookie-policy :netscape netscape-cookie-policy 152 | [_] CookieSpecs/NETSCAPE) 153 | (defmethod get-cookie-policy :standard standard-cookie-policy 154 | [_] CookieSpecs/STANDARD) 155 | (defmethod get-cookie-policy :stardard-strict standard-strict-cookie-policy 156 | [_] CookieSpecs/STANDARD_STRICT) 157 | 158 | (defn request-config [{:keys [connection-timeout 159 | connection-request-timeout 160 | socket-timeout 161 | max-redirects 162 | cookie-spec 163 | normalize-uri 164 | ; deprecated 165 | conn-request-timeout 166 | conn-timeout] 167 | :as req}] 168 | (let [config (-> (RequestConfig/custom) 169 | (.setConnectTimeout (or connection-timeout conn-timeout -1)) 170 | (.setSocketTimeout (or socket-timeout -1)) 171 | (.setConnectionRequestTimeout 172 | (or connection-request-timeout conn-request-timeout -1)) 173 | (.setRedirectsEnabled true) 174 | (.setCircularRedirectsAllowed 175 | (boolean (opt req :allow-circular-redirects))) 176 | (.setRelativeRedirectsAllowed 177 | ((complement false?) 178 | (opt req :allow-relative-redirects))))] 179 | (if cookie-spec 180 | (.setCookieSpec config CUSTOM_COOKIE_POLICY) 181 | (.setCookieSpec config (get-cookie-policy req))) 182 | (when max-redirects (.setMaxRedirects config max-redirects)) 183 | (when-not (nil? normalize-uri) (.setNormalizeUri config normalize-uri)) 184 | (.build config))) 185 | 186 | (defmulti ^:private construct-http-host (fn [proxy-host proxy-port] 187 | (class proxy-host))) 188 | (defmethod construct-http-host String 189 | [^String proxy-host ^Long proxy-port] 190 | (if proxy-port 191 | (HttpHost. proxy-host proxy-port) 192 | (HttpHost. proxy-host))) 193 | (defmethod construct-http-host java.net.InetAddress 194 | [^InetAddress proxy-host ^Long proxy-port] 195 | (if proxy-port 196 | (HttpHost. proxy-host proxy-port) 197 | (HttpHost. proxy-host))) 198 | 199 | (defn ^HttpRoutePlanner get-route-planner 200 | "Return an HttpRoutePlanner that either use the supplied proxy settings 201 | if any, or the JVM/system proxy settings otherwise" 202 | [^String proxy-host ^Long proxy-port proxy-ignore-hosts http-url] 203 | (let [ignore-proxy? (and http-url 204 | (contains? (set proxy-ignore-hosts) 205 | (.getHost (URL. http-url))))] 206 | (if (and proxy-host (not ignore-proxy?)) 207 | (DefaultProxyRoutePlanner. (construct-http-host proxy-host proxy-port)) 208 | (SystemDefaultRoutePlanner. (ProxySelector/getDefault))))) 209 | 210 | (defn build-cache-config 211 | "Given a request with :cache-config as a map or a CacheConfig object, return a 212 | CacheConfig object, or nil if no cache config is found. If :cache-config is a 213 | map, it checks for the following options: 214 | - :allow-303-caching 215 | - :asynchronous-worker-idle-lifetime-secs 216 | - :asynchronous-workers-core 217 | - :asynchronous-workers-max 218 | - :heuristic-caching-enabled 219 | - :heuristic-coefficient 220 | - :heuristic-default-lifetime 221 | - :max-cache-entries 222 | - :max-object-size 223 | - :max-update-retries 224 | - :never-cache-http10-responses-with-query-string 225 | - :revalidation-queue-size 226 | - :shared-cache 227 | - :weak-etag-on-put-delete-allowed" 228 | [request] 229 | (when-let [cc (:cache-config request)] 230 | (if (instance? CacheConfig cc) 231 | cc 232 | (let [config (CacheConfig/custom) 233 | {:keys [allow-303-caching 234 | asynchronous-worker-idle-lifetime-secs 235 | asynchronous-workers-core 236 | asynchronous-workers-max 237 | heuristic-caching-enabled 238 | heuristic-coefficient 239 | heuristic-default-lifetime 240 | max-cache-entries 241 | max-object-size 242 | max-update-retries 243 | never-cache-http10-responses-with-query-string 244 | revalidation-queue-size 245 | shared-cache 246 | weak-etag-on-put-delete-allowed]} cc] 247 | (when (instance? Boolean allow-303-caching) 248 | (.setAllow303Caching config allow-303-caching)) 249 | (when asynchronous-worker-idle-lifetime-secs 250 | (.setAsynchronousWorkerIdleLifetimeSecs 251 | config asynchronous-worker-idle-lifetime-secs)) 252 | (when asynchronous-workers-core 253 | (.setAsynchronousWorkersCore config asynchronous-workers-core)) 254 | (when asynchronous-workers-max 255 | (.setAsynchronousWorkersMax config asynchronous-workers-max)) 256 | (when (instance? Boolean heuristic-caching-enabled) 257 | (.setHeuristicCachingEnabled config heuristic-caching-enabled)) 258 | (when heuristic-coefficient 259 | (.setHeuristicCoefficient config heuristic-coefficient)) 260 | (when heuristic-default-lifetime 261 | (.setHeuristicDefaultLifetime config heuristic-default-lifetime)) 262 | (when max-cache-entries 263 | (.setMaxCacheEntries config max-cache-entries)) 264 | (when max-object-size 265 | (.setMaxObjectSize config max-object-size)) 266 | (when max-update-retries 267 | (.setMaxUpdateRetries config max-update-retries)) 268 | ;; I would add this option, but there is a bug in 4.x CacheConfig that 269 | ;; it does not actually correctly use the object from the builder. 270 | ;; It's fixed in 5.0 however 271 | ;; (when (boolean? never-cache-http10-responses-with-query-string) 272 | ;; (.setNeverCacheHTTP10ResponsesWithQueryString 273 | ;; config never-cache-http10-responses-with-query-string)) 274 | (when revalidation-queue-size 275 | (.setRevalidationQueueSize config revalidation-queue-size)) 276 | (when (instance? Boolean shared-cache) 277 | (.setSharedCache config shared-cache)) 278 | (when (instance? Boolean weak-etag-on-put-delete-allowed) 279 | (.setWeakETagOnPutDeleteAllowed config weak-etag-on-put-delete-allowed)) 280 | (.build config))))) 281 | 282 | (defn build-http-client 283 | "Builds an Apache `HttpClient` from a clj-http request map. Optional arguments 284 | `http-url` and `proxy-ignore-hosts` are used to specify the host and a list of 285 | hostnames to ignore for any proxy settings. They can be safely ignored if not 286 | using proxies." 287 | [{:keys [retry-handler request-interceptor 288 | response-interceptor proxy-host proxy-port 289 | http-builder-fns cookie-spec 290 | cookie-policy-registry 291 | ^HttpClientBuilder http-client-builder] 292 | :as req} 293 | caching? 294 | conn-mgr 295 | & [http-url proxy-ignore-hosts]] 296 | ;; have to let first, otherwise we get a reflection warning on (.build) 297 | (let [cache? (opt req :cache) 298 | builder (-> (cond 299 | http-client-builder http-client-builder 300 | caching? 301 | ^HttpClientBuilder (CachingHttpClientBuilder/create) 302 | :else 303 | ^HttpClientBuilder (HttpClients/custom)) 304 | (.setConnectionManager conn-mgr) 305 | (.setRedirectStrategy 306 | (get-redirect-strategy req)) 307 | (add-retry-handler retry-handler) 308 | 309 | ;; prefer using clj-http.client/wrap-decompression 310 | ;; for consistency between sync/async client options 311 | (.disableContentCompression) 312 | 313 | ;; By default, get the proxy settings 314 | ;; from the jvm or system properties 315 | (.setRoutePlanner 316 | (get-route-planner 317 | proxy-host proxy-port 318 | proxy-ignore-hosts http-url)))] 319 | (when cache? 320 | (.setCacheConfig ^CachingHttpClientBuilder builder (build-cache-config req))) 321 | (when (or cookie-policy-registry cookie-spec) 322 | (if cookie-policy-registry 323 | ;; They have a custom registry they'd like to re-use, so use that 324 | (.setDefaultCookieSpecRegistry builder cookie-policy-registry) 325 | ;; They have only a one-time function for cookie spec, so use that 326 | (.setDefaultCookieSpecRegistry 327 | builder (create-custom-cookie-policy-registry cookie-spec)))) 328 | (when request-interceptor 329 | (.addInterceptorLast 330 | builder (proxy [HttpRequestInterceptor] [] 331 | (process [req ctx] 332 | (request-interceptor req ctx))))) 333 | 334 | (when response-interceptor 335 | (.addInterceptorLast 336 | builder (proxy [HttpResponseInterceptor] [] 337 | (process [resp ctx] 338 | (response-interceptor 339 | resp ctx))))) 340 | (doseq [http-builder-fn http-builder-fns] 341 | (http-builder-fn builder req)) 342 | (.build builder))) 343 | 344 | (defn build-async-http-client 345 | "Builds an Apache `HttpAsyncClient` from a clj-http request map. Optional 346 | arguments `http-url` and `proxy-ignore-hosts` are used to specify the host 347 | and a list of hostnames to ignore for any proxy settings. They can be safely 348 | ignored if not using proxies." 349 | [{:keys [request-interceptor response-interceptor 350 | proxy-host proxy-port async-http-builder-fns] 351 | :as req} 352 | conn-mgr & [http-url proxy-ignore-hosts]] 353 | ;; have to let first, otherwise we get a reflection warning on (.build) 354 | (let [^HttpAsyncClientBuilder builder (-> (HttpAsyncClients/custom) 355 | (.setConnectionManager conn-mgr) 356 | (.setRedirectStrategy 357 | (get-redirect-strategy req)) 358 | ;; By default, get the proxy 359 | ;; settings from the jvm or system 360 | ;; properties 361 | (.setRoutePlanner 362 | (get-route-planner 363 | proxy-host proxy-port 364 | proxy-ignore-hosts http-url)))] 365 | (when (conn/reusable? conn-mgr) 366 | (.setConnectionManagerShared builder true)) 367 | 368 | (when request-interceptor 369 | (.addInterceptorLast 370 | builder (proxy [HttpRequestInterceptor] [] 371 | (process [req ctx] 372 | (request-interceptor req ctx))))) 373 | 374 | (when response-interceptor 375 | (.addInterceptorLast 376 | builder (proxy [HttpResponseInterceptor] [] 377 | (process [resp ctx] 378 | (response-interceptor 379 | resp ctx))))) 380 | (doseq [async-http-builder-fn async-http-builder-fns] 381 | (async-http-builder-fn builder req)) 382 | (.build builder))) 383 | 384 | (defn http-get [] 385 | (HttpGet. "https://www.google.com")) 386 | 387 | (defn make-proxy-method-with-body 388 | [method] 389 | (fn [url] 390 | (doto (proxy [HttpEntityEnclosingRequestBase] [] 391 | (getMethod [] (.toUpperCase (name method) Locale/ROOT))) 392 | (.setURI (URI. url))))) 393 | 394 | (def proxy-head-with-body (make-proxy-method-with-body :head)) 395 | (def proxy-delete-with-body (make-proxy-method-with-body :delete)) 396 | (def proxy-get-with-body (make-proxy-method-with-body :get)) 397 | (def proxy-copy-with-body (make-proxy-method-with-body :copy)) 398 | (def proxy-move-with-body (make-proxy-method-with-body :move)) 399 | (def proxy-patch-with-body (make-proxy-method-with-body :patch)) 400 | 401 | (def ^:dynamic *cookie-store* nil) 402 | 403 | (defn make-proxy-method [method url] 404 | (doto (proxy [HttpRequestBase] [] 405 | (getMethod 406 | [] 407 | (str method))) 408 | (.setURI (URI/create url)))) 409 | 410 | (defn http-request-for 411 | "Provides the HttpRequest object for a particular request-method and url" 412 | [request-method ^String http-url body] 413 | (case request-method 414 | :get (if body 415 | (proxy-get-with-body http-url) 416 | (HttpGet. http-url)) 417 | :head (if body 418 | (proxy-head-with-body http-url) 419 | (HttpHead. http-url)) 420 | :put (HttpPut. http-url) 421 | :post (HttpPost. http-url) 422 | :options (HttpOptions. http-url) 423 | :delete (if body 424 | (proxy-delete-with-body http-url) 425 | (HttpDelete. http-url)) 426 | :copy (proxy-copy-with-body http-url) 427 | :move (proxy-move-with-body http-url) 428 | :patch (if body 429 | (proxy-patch-with-body http-url) 430 | (HttpPatch. http-url)) 431 | (if body 432 | ((make-proxy-method-with-body request-method) http-url) 433 | (make-proxy-method request-method http-url)))) 434 | 435 | (defn ^HttpClientContext http-context [caching? request-config http-client-context] 436 | (let [^HttpClientContext typed-context (or http-client-context 437 | (if caching? 438 | (HttpCacheContext/create) 439 | (HttpClientContext/create)))] 440 | (doto typed-context 441 | (.setRequestConfig request-config)))) 442 | 443 | (defn ^CredentialsProvider credentials-provider [] 444 | (BasicCredentialsProvider.)) 445 | 446 | (defn- coerce-body-entity 447 | "Coerce the http-entity from an HttpResponse to a stream that closes itself 448 | and the connection manager when closed." 449 | [^HttpEntity http-entity conn-mgr ^CloseableHttpResponse response] 450 | (if http-entity 451 | (proxy [FilterInputStream] 452 | [^InputStream (.getContent http-entity)] 453 | (close [] 454 | (try 455 | ;; Eliminate the reflection warning from proxy-super 456 | (let [^InputStream this this] 457 | (proxy-super close)) 458 | (finally 459 | (when (instance? CloseableHttpResponse response) 460 | (.close response)) 461 | (when-not (conn/reusable? conn-mgr) 462 | (conn/shutdown-manager conn-mgr)))))) 463 | (when-not (conn/reusable? conn-mgr) 464 | (conn/shutdown-manager conn-mgr)))) 465 | 466 | (defn- print-debug! 467 | "Print out debugging information to *out* for a given request." 468 | [{:keys [debug-body body] :as req} http-req] 469 | (println 470 | (with-out-str 471 | (println "Request:" (type body)) 472 | (clojure.pprint/pprint 473 | (assoc req 474 | :body (if (opt req :debug-body) 475 | (cond 476 | (isa? (type body) String) 477 | body 478 | 479 | (isa? (type body) HttpEntity) 480 | (let [baos (ByteArrayOutputStream.)] 481 | (.writeTo ^HttpEntity body baos) 482 | (.toString baos "UTF-8")) 483 | 484 | :else nil) 485 | (if (isa? (type body) String) 486 | (format "... %s bytes ..." 487 | (count body)) 488 | (and body (bean body)))) 489 | :body-type (type body))) 490 | (println "HttpRequest:") 491 | (clojure.pprint/pprint (bean http-req))))) 492 | 493 | (defn- build-response-map 494 | [^HttpResponse response req ^HttpUriRequest http-req http-url 495 | conn-mgr ^HttpClientContext context ^CloseableHttpClient client] 496 | (let [^HttpEntity entity (.getEntity response) 497 | status (.getStatusLine response) 498 | protocol-version (.getProtocolVersion status) 499 | body (:body req) 500 | response 501 | {:body (coerce-body-entity entity conn-mgr response) 502 | :http-client client 503 | :headers (parse-headers 504 | (.headerIterator response) 505 | (opt req :use-header-maps-in-response)) 506 | :length (if (nil? entity) 0 (.getContentLength entity)) 507 | :chunked? (if (nil? entity) false (.isChunked entity)) 508 | :repeatable? (if (nil? entity) false (.isRepeatable entity)) 509 | :streaming? (if (nil? entity) false (.isStreaming entity)) 510 | :status (.getStatusCode status) 511 | :protocol-version {:name (.getProtocol protocol-version) 512 | :major (.getMajor protocol-version) 513 | :minor (.getMinor protocol-version)} 514 | :reason-phrase (.getReasonPhrase status) 515 | :trace-redirects (mapv str (.getRedirectLocations context)) 516 | :cached (when (instance? HttpCacheContext context) 517 | (when-let [cache-resp (.getCacheResponseStatus ^HttpCacheContext context)] 518 | (-> cache-resp str keyword)))}] 519 | (if (opt req :save-request) 520 | (-> response 521 | (assoc :request req) 522 | (assoc-in [:request :body-type] (type body)) 523 | (assoc-in [:request :http-url] http-url) 524 | (update-in [:request] 525 | #(if (opt req :debug-body) 526 | (assoc % :body-content 527 | (cond 528 | (isa? (type (:body %)) String) 529 | (:body %) 530 | 531 | (isa? (type (:body %)) HttpEntity) 532 | (let [baos (ByteArrayOutputStream.)] 533 | (.writeTo ^HttpEntity (:body %) baos) 534 | (.toString baos "UTF-8")) 535 | 536 | :else nil)) 537 | %)) 538 | (assoc-in [:request :http-req] http-req)) 539 | response))) 540 | 541 | (defn- get-conn-mgr 542 | [async? req] 543 | (if async? 544 | (or conn/*async-connection-manager* 545 | (conn/make-regular-async-conn-manager req)) 546 | (or conn/*connection-manager* 547 | (conn/make-regular-conn-manager req)))) 548 | 549 | (defn request 550 | ([req] (request req nil nil)) 551 | ([{:keys [body connection-timeout connection-request-timeout connection-manager 552 | cookie-store cookie-policy headers multipart query-string 553 | redirect-strategy max-redirects retry-handler 554 | request-method scheme server-name server-port socket-timeout 555 | uri response-interceptor proxy-host proxy-port 556 | http-client-context http-request-config http-client 557 | proxy-ignore-hosts proxy-user proxy-pass digest-auth ntlm-auth 558 | multipart-mode multipart-charset 559 | ; deprecated 560 | conn-timeout conn-request-timeout] 561 | :as req} respond raise] 562 | (let [async? (opt req :async) 563 | cache? (opt req :cache) 564 | scheme (name scheme) 565 | http-url (str scheme "://" server-name 566 | (when server-port (str ":" server-port)) 567 | uri 568 | (when query-string (str "?" query-string))) 569 | conn-mgr (or connection-manager 570 | (get-conn-mgr async? req)) 571 | proxy-ignore-hosts (or proxy-ignore-hosts 572 | #{"localhost" "127.0.0.1"}) 573 | ^RequestConfig request-config (or http-request-config 574 | (request-config req)) 575 | ^HttpClientContext context 576 | (http-context cache? request-config http-client-context) 577 | ^HttpUriRequest http-req (http-request-for 578 | request-method http-url body)] 579 | (when-not (conn/reusable? conn-mgr) 580 | (.addHeader http-req "Connection" "close")) 581 | (when-let [cookie-jar (or cookie-store *cookie-store*)] 582 | (.setCookieStore context cookie-jar)) 583 | (when-let [[user pass] digest-auth] 584 | (.setCredentialsProvider 585 | context 586 | (doto (credentials-provider) 587 | (.setCredentials (AuthScope. nil -1 nil) 588 | (UsernamePasswordCredentials. user pass))))) 589 | (when-let [[user password host domain] ntlm-auth] 590 | (.setCredentialsProvider 591 | context 592 | (doto (credentials-provider) 593 | (.setCredentials (AuthScope. nil -1 nil) 594 | (NTCredentials. user password host domain))))) 595 | (when (and proxy-user proxy-pass) 596 | (let [authscope (AuthScope. proxy-host proxy-port) 597 | creds (UsernamePasswordCredentials. proxy-user proxy-pass)] 598 | (.setCredentialsProvider 599 | context 600 | (doto (credentials-provider) 601 | (.setCredentials authscope creds))))) 602 | (if multipart 603 | (.setEntity ^HttpEntityEnclosingRequest http-req 604 | (mp/create-multipart-entity multipart req)) 605 | (when (and body (instance? HttpEntityEnclosingRequest http-req)) 606 | (if (instance? HttpEntity body) 607 | (.setEntity ^HttpEntityEnclosingRequest http-req body) 608 | (.setEntity ^HttpEntityEnclosingRequest http-req 609 | (if (string? body) 610 | (StringEntity. ^String body "UTF-8") 611 | (ByteArrayEntity. body)))))) 612 | (doseq [[header-n header-v] headers] 613 | (if (coll? header-v) 614 | (doseq [header-vth header-v] 615 | (.addHeader http-req header-n header-vth)) 616 | (.addHeader http-req header-n (str header-v)))) 617 | (when (opt req :debug) (print-debug! req http-req)) 618 | (if-not async? 619 | (let [^CloseableHttpClient client 620 | (or http-client 621 | (build-http-client req cache? 622 | conn-mgr http-url proxy-ignore-hosts))] 623 | (try 624 | (build-response-map (.execute client http-req context) 625 | req http-req http-url conn-mgr context client) 626 | (catch Throwable t 627 | (when-not (conn/reusable? conn-mgr) 628 | (conn/shutdown-manager conn-mgr)) 629 | (throw t)))) 630 | (let [^CloseableHttpAsyncClient client 631 | (or http-client 632 | (build-async-http-client req conn-mgr http-url proxy-ignore-hosts)) 633 | original-thread-bindings (clojure.lang.Var/getThreadBindingFrame)] 634 | (when cache? 635 | (throw (IllegalArgumentException. 636 | "caching is not yet supported for async clients"))) 637 | (.start client) 638 | (.execute client http-req context 639 | (reify org.apache.http.concurrent.FutureCallback 640 | (failed [this ex] 641 | (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings) 642 | (when-not (conn/reusable? conn-mgr) 643 | (conn/shutdown-manager conn-mgr)) 644 | (if (opt req :ignore-unknown-host) 645 | ((:unknown-host-respond req) nil) 646 | (raise ex))) 647 | (completed [this resp] 648 | (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings) 649 | (try 650 | (respond (build-response-map 651 | resp req http-req http-url 652 | conn-mgr context client)) 653 | (catch Throwable t 654 | (when-not (conn/reusable? conn-mgr) 655 | (conn/shutdown-manager conn-mgr)) 656 | (raise t)))) 657 | (cancelled [this] 658 | (clojure.lang.Var/resetThreadBindingFrame original-thread-bindings) 659 | ;; Run the :oncancel function if available 660 | (when-let [oncancel (:oncancel req)] 661 | (oncancel)) 662 | ;; Attempt to abort the execution of the request 663 | (.abort http-req) 664 | (when-not (conn/reusable? conn-mgr) 665 | (conn/shutdown-manager conn-mgr)))))))))) 666 | -------------------------------------------------------------------------------- /src/clj_http/core_old.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.core-old 2 | "Core HTTP request/response implementation." 3 | (:require [clj-http.conn-mgr :as conn] 4 | [clj-http.headers :as headers] 5 | [clj-http.multipart :as mp] 6 | [clj-http.util :refer [opt]] 7 | clojure.pprint) 8 | (:import [java.io ByteArrayOutputStream FilterInputStream InputStream] 9 | java.net.URI 10 | [org.apache.http HeaderIterator HttpEntity HttpEntityEnclosingRequest HttpHost HttpResponseInterceptor] 11 | [org.apache.http.auth AuthScope NTCredentials UsernamePasswordCredentials] 12 | [org.apache.http.client HttpClient HttpRequestRetryHandler] 13 | [org.apache.http.client.methods HttpDelete HttpEntityEnclosingRequestBase HttpGet HttpHead HttpOptions HttpPatch HttpPost HttpPut HttpUriRequest] 14 | [org.apache.http.client.params ClientPNames CookiePolicy] 15 | org.apache.http.conn.ClientConnectionManager 16 | org.apache.http.conn.params.ConnRoutePNames 17 | org.apache.http.conn.routing.HttpRoute 18 | org.apache.http.cookie.CookieSpecFactory 19 | org.apache.http.cookie.params.CookieSpecPNames 20 | [org.apache.http.entity ByteArrayEntity StringEntity] 21 | org.apache.http.impl.client.DefaultHttpClient 22 | org.apache.http.impl.conn.ProxySelectorRoutePlanner 23 | org.apache.http.impl.cookie.BrowserCompatSpec 24 | org.apache.http.params.CoreConnectionPNames)) 25 | 26 | (defn parse-headers 27 | "Takes a HeaderIterator and returns a map of names to values. 28 | 29 | If a name appears more than once (like `set-cookie`) then the value 30 | will be a vector containing the values in the order they appeared 31 | in the headers." 32 | [^HeaderIterator headers & [use-header-maps-in-response?]] 33 | (if-not use-header-maps-in-response? 34 | (->> (headers/header-iterator-seq headers) 35 | (map (fn [[k v]] 36 | [(.toLowerCase ^String k) v])) 37 | (reduce (fn [hs [k v]] 38 | (headers/assoc-join hs k v)) 39 | {})) 40 | (->> (headers/header-iterator-seq headers) 41 | (reduce (fn [hs [k v]] 42 | (headers/assoc-join hs k v)) 43 | (headers/header-map))))) 44 | 45 | (defn set-client-param [^HttpClient client key val] 46 | (when-not (nil? val) 47 | (-> client 48 | (.getParams) 49 | (.setParameter key val)))) 50 | 51 | (defn make-proxy-method-with-body 52 | [method] 53 | (fn [^String url] 54 | (doto (proxy [HttpEntityEnclosingRequestBase] [] 55 | (getMethod [] (.toUpperCase (name method)))) 56 | (.setURI (URI. url))))) 57 | 58 | (def proxy-delete-with-body (make-proxy-method-with-body :delete)) 59 | (def proxy-get-with-body (make-proxy-method-with-body :get)) 60 | (def proxy-copy-with-body (make-proxy-method-with-body :copy)) 61 | (def proxy-move-with-body (make-proxy-method-with-body :move)) 62 | (def proxy-patch-with-body (make-proxy-method-with-body :patch)) 63 | 64 | (def ^:dynamic *cookie-store* nil) 65 | 66 | (defn- set-routing 67 | "Use ProxySelectorRoutePlanner to choose proxy sensible based on 68 | http.nonProxyHosts" 69 | [^DefaultHttpClient client] 70 | (.setRoutePlanner client 71 | (ProxySelectorRoutePlanner. 72 | (.. client getConnectionManager getSchemeRegistry) nil)) 73 | client) 74 | 75 | (defn maybe-force-proxy [^DefaultHttpClient client 76 | ^HttpEntityEnclosingRequestBase request 77 | proxy-host proxy-port proxy-ignore-hosts] 78 | (let [uri (.getURI request)] 79 | (when (and (nil? ((set proxy-ignore-hosts) (.getHost uri))) proxy-host) 80 | (let [target (HttpHost. (.getHost uri) (.getPort uri) (.getScheme uri)) 81 | route (HttpRoute. target nil (HttpHost. ^String proxy-host 82 | (int proxy-port)) 83 | (.. client getConnectionManager getSchemeRegistry 84 | (getScheme target) isLayered))] 85 | (set-client-param client ConnRoutePNames/FORCED_ROUTE route))) 86 | request)) 87 | 88 | (defn cookie-spec 89 | "Create an instance of a 90 | org.apache.http.impl.cookie.BrowserCompatSpec with a validate 91 | function that you pass in. This function takes two parameters, a 92 | cookie and an origin." 93 | [f] 94 | (proxy [BrowserCompatSpec] [] 95 | (validate [cookie origin] (f cookie origin)))) 96 | 97 | (defn cookie-spec-factory 98 | "Create an instance of a org.apache.http.cookie.CookieSpecFactory 99 | with a newInstance implementation that returns a cookie 100 | specification with a validate function that you pass in. The 101 | function takes two parameters: cookie and origin." 102 | [f] 103 | (proxy 104 | [CookieSpecFactory] [] 105 | (newInstance [params] (cookie-spec f)))) 106 | 107 | (defn add-client-params! 108 | "Add various client params to the http-client object, if needed." 109 | [^DefaultHttpClient http-client kvs] 110 | (let [cookie-policy (:cookie-policy kvs) 111 | cookie-policy-name (str (type cookie-policy)) 112 | kvs (dissoc kvs :cookie-policy)] 113 | (when cookie-policy 114 | (-> http-client 115 | .getCookieSpecs 116 | (.register cookie-policy-name (cookie-spec-factory cookie-policy)))) 117 | (doto http-client 118 | (set-client-param ClientPNames/COOKIE_POLICY 119 | (if cookie-policy 120 | cookie-policy-name 121 | CookiePolicy/BROWSER_COMPATIBILITY)) 122 | (set-client-param CookieSpecPNames/SINGLE_COOKIE_HEADER true) 123 | (set-client-param ClientPNames/HANDLE_REDIRECTS false)) 124 | 125 | (doseq [[k v] kvs] 126 | (set-client-param http-client 127 | k (cond 128 | (and (not= ClientPNames/CONN_MANAGER_TIMEOUT k) 129 | (instance? Long v)) 130 | (Integer. ^Long v) 131 | true v))))) 132 | 133 | (defn- coerce-body-entity 134 | "Coerce the http-entity from an HttpResponse to either a byte-array, or a 135 | stream that closes itself and the connection manager when closed." 136 | [{:keys [as]} ^HttpEntity http-entity ^ClientConnectionManager conn-mgr] 137 | (if http-entity 138 | (proxy [FilterInputStream] 139 | [^InputStream (.getContent http-entity)] 140 | (close [] 141 | (try 142 | ;; Eliminate the reflection warning from proxy-super 143 | (let [^InputStream this this] 144 | (proxy-super close)) 145 | (finally 146 | (when-not (conn/reusable? conn-mgr) 147 | (conn/shutdown-manager conn-mgr)))))) 148 | (when-not (conn/reusable? conn-mgr) 149 | (conn/shutdown-manager conn-mgr)))) 150 | 151 | (defn- print-debug! 152 | "Print out debugging information to *out* for a given request." 153 | [{:keys [debug-body body] :as req} http-req] 154 | (println "Request:" (type body)) 155 | (clojure.pprint/pprint 156 | (assoc req 157 | :body (if (opt req :debug-body) 158 | (cond 159 | (isa? (type body) String) 160 | body 161 | 162 | (isa? (type body) HttpEntity) 163 | (let [baos (ByteArrayOutputStream.)] 164 | (.writeTo ^HttpEntity body baos) 165 | (.toString baos "UTF-8")) 166 | 167 | :else nil) 168 | (if (isa? (type body) String) 169 | (format "... %s bytes ..." 170 | (count body)) 171 | (and body (bean body)))) 172 | :body-type (type body))) 173 | (println "HttpRequest:") 174 | (clojure.pprint/pprint (bean http-req))) 175 | 176 | (defn http-request-for 177 | "Provides the HttpRequest object for a particular request-method and url" 178 | [request-method ^String http-url body] 179 | (case request-method 180 | :get (if body 181 | (proxy-get-with-body http-url) 182 | (HttpGet. http-url)) 183 | :head (HttpHead. http-url) 184 | :put (HttpPut. http-url) 185 | :post (HttpPost. http-url) 186 | :options (HttpOptions. http-url) 187 | :delete (if body 188 | (proxy-delete-with-body http-url) 189 | (HttpDelete. http-url)) 190 | :copy (proxy-copy-with-body http-url) 191 | :move (proxy-move-with-body http-url) 192 | :patch (if body 193 | (proxy-patch-with-body http-url) 194 | (HttpPatch. http-url)) 195 | (throw (IllegalArgumentException. 196 | (str "Invalid request method " request-method))))) 197 | 198 | (defn request 199 | "Executes the HTTP request corresponding to the given Ring request map and 200 | returns the Ring response map corresponding to the resulting HTTP response. 201 | 202 | Note that where Ring uses InputStreams for the request and response bodies, 203 | the clj-http uses ByteArrays for the bodies." 204 | [{:keys [request-method scheme server-name server-port uri query-string 205 | headers body multipart socket-timeout connection-timeout proxy-host 206 | proxy-ignore-hosts proxy-port proxy-user proxy-pass as cookie-store 207 | retry-handler response-interceptor digest-auth ntlm-auth 208 | connection-manager client-params 209 | ; deprecated 210 | conn-timeout 211 | ] 212 | :as req}] 213 | (let [^ClientConnectionManager conn-mgr 214 | (or connection-manager 215 | conn/*connection-manager* 216 | (conn/make-regular-conn-manager req)) 217 | ^DefaultHttpClient http-client (set-routing 218 | (DefaultHttpClient. conn-mgr)) 219 | scheme (name scheme)] 220 | (when-let [cookie-store (or cookie-store *cookie-store*)] 221 | (.setCookieStore http-client cookie-store)) 222 | (when retry-handler 223 | (.setHttpRequestRetryHandler 224 | http-client 225 | (proxy [HttpRequestRetryHandler] [] 226 | (retryRequest [e cnt context] 227 | (retry-handler e cnt context))))) 228 | (add-client-params! 229 | http-client 230 | ;; merge in map of specified timeouts, to 231 | ;; support backward compatibility. 232 | (merge {CoreConnectionPNames/SO_TIMEOUT socket-timeout 233 | CoreConnectionPNames/CONNECTION_TIMEOUT (or connection-timeout 234 | conn-timeout)} 235 | client-params)) 236 | 237 | (when-let [[user pass] digest-auth] 238 | (.setCredentials 239 | (.getCredentialsProvider http-client) 240 | (AuthScope. nil -1 nil) 241 | (UsernamePasswordCredentials. user pass))) 242 | (when-let [[user password host domain] ntlm-auth] 243 | (.setCredentials 244 | (.getCredentialsProvider http-client) 245 | (AuthScope. nil -1 nil) 246 | (NTCredentials. user password host domain))) 247 | 248 | (when (and proxy-user proxy-pass) 249 | (let [authscope (AuthScope. proxy-host proxy-port) 250 | creds (UsernamePasswordCredentials. proxy-user proxy-pass)] 251 | (.setCredentials (.getCredentialsProvider http-client) 252 | authscope creds))) 253 | (let [http-url (str scheme "://" server-name 254 | (when server-port (str ":" server-port)) 255 | uri 256 | (when query-string (str "?" query-string))) 257 | req (assoc req :http-url http-url) 258 | proxy-ignore-hosts (or proxy-ignore-hosts 259 | #{"localhost" "127.0.0.1"}) 260 | ^HttpUriRequest http-req (maybe-force-proxy 261 | http-client 262 | (http-request-for request-method 263 | http-url body) 264 | proxy-host 265 | proxy-port 266 | proxy-ignore-hosts)] 267 | (when response-interceptor 268 | (.addResponseInterceptor 269 | http-client 270 | (proxy [HttpResponseInterceptor] [] 271 | (process [resp ctx] 272 | (response-interceptor resp ctx))))) 273 | (when-not (conn/reusable? conn-mgr) 274 | (.addHeader http-req "Connection" "close")) 275 | (doseq [[header-n header-v] headers] 276 | (if (coll? header-v) 277 | (doseq [header-vth header-v] 278 | (.addHeader http-req header-n header-vth)) 279 | (.addHeader http-req header-n (str header-v)))) 280 | (if multipart 281 | (.setEntity ^HttpEntityEnclosingRequest http-req 282 | (mp/create-multipart-entity multipart req)) 283 | (when (and body (instance? HttpEntityEnclosingRequest http-req)) 284 | (if (instance? HttpEntity body) 285 | (.setEntity ^HttpEntityEnclosingRequest http-req body) 286 | (.setEntity ^HttpEntityEnclosingRequest http-req 287 | (if (string? body) 288 | (StringEntity. ^String body "UTF-8") 289 | (ByteArrayEntity. body)))))) 290 | (when (opt req :debug) (print-debug! req http-req)) 291 | (try 292 | (let [http-resp (.execute http-client http-req) 293 | http-entity (.getEntity http-resp) 294 | resp {:status (.getStatusCode (.getStatusLine http-resp)) 295 | :headers (parse-headers 296 | (.headerIterator http-resp) 297 | (opt req :use-header-maps-in-response)) 298 | :body (coerce-body-entity req http-entity conn-mgr)}] 299 | (if (opt req :save-request) 300 | (-> resp 301 | (assoc :request req) 302 | (assoc-in [:request :body-type] (type body)) 303 | (update-in [:request] 304 | #(if (opt req :debug-body) 305 | (assoc % :body-content 306 | (cond 307 | (isa? (type (:body %)) String) 308 | (:body %) 309 | 310 | (isa? (type (:body %)) HttpEntity) 311 | (let [baos (ByteArrayOutputStream.)] 312 | (.writeTo ^HttpEntity (:body %) baos) 313 | (.toString baos "UTF-8")) 314 | 315 | :else nil)) 316 | %)) 317 | (assoc-in [:request :http-req] http-req) 318 | (dissoc :save-request?)) 319 | resp)) 320 | (catch Throwable e 321 | (when-not (conn/reusable? conn-mgr) 322 | (conn/shutdown-manager conn-mgr)) 323 | (throw e)))))) 324 | -------------------------------------------------------------------------------- /src/clj_http/headers.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.headers 2 | "Provides wrap-header-map, which is middleware allows headers to be 3 | specified more flexibly. In requests and responses, headers can be 4 | accessed as strings or keywords of any case. In requests, string 5 | header names will be sent to the server with their casing unchanged, 6 | while keyword header names will be transformed into their canonical 7 | HTTP representation (e.g. :accept-encoding will become 8 | \"Accept-Encoding\")." 9 | (:require [clojure.string :as s] 10 | [potemkin :as potemkin]) 11 | (:import java.util.Locale 12 | [org.apache.http Header HeaderIterator])) 13 | 14 | (def special-cases 15 | "A collection of HTTP headers that do not follow the normal 16 | Looks-Like-This casing." 17 | ["Content-MD5" 18 | "DNT" 19 | "ETag" 20 | "P3P" 21 | "TE" 22 | "WWW-Authenticate" 23 | "X-ATT-DeviceId" 24 | "X-UA-Compatible" 25 | "X-WebKit-CSP" 26 | "X-XSS-Protection"]) 27 | 28 | (defn special-case 29 | "Returns the special-case capitalized version of a string if that 30 | string is a special case, otherwise returns the string unchanged." 31 | [^String s] 32 | (or (first (filter #(.equalsIgnoreCase ^String % s) special-cases)) 33 | s)) 34 | 35 | (defn ^String lower-case 36 | "Converts a string to all lower-case, using the root locale. 37 | 38 | Warning: This is not a general purpose lower-casing function -- it 39 | is useful for case-insensitive comparisons of strings, not for 40 | converting a string into something that's useful for humans." 41 | [^CharSequence s] 42 | (when s 43 | (.toLowerCase (.toString s) Locale/ROOT))) 44 | 45 | (defn title-case 46 | "Converts a character to titlecase." 47 | [^Character c] 48 | (when c 49 | (Character/toTitleCase c))) 50 | 51 | (defn canonicalize 52 | "Transforms a keyword header name into its canonical string 53 | representation. 54 | 55 | The canonical string representation is title-cased words separated 56 | by dashes, like so: :date -> \"Date\", :DATE -> \"Date\", and 57 | :foo-bar -> \"Foo-Bar\". 58 | 59 | However, there is special-casing for some common headers, so: :p3p 60 | -> \"P3P\", and :content-md5 -> \"Content-MD5\"." 61 | [k] 62 | (when k 63 | (-> (name k) 64 | (lower-case) 65 | (s/replace #"(?:^.|-.)" 66 | (fn [s] 67 | (if (next s) 68 | (str (first s) 69 | (title-case (second s))) 70 | (str (title-case (first s)))))) 71 | (special-case)))) 72 | 73 | (defn normalize 74 | "Turns a string or keyword into normalized form, which is a 75 | lowercase string." 76 | [k] 77 | (when k 78 | (lower-case (name k)))) 79 | 80 | (defn header-iterator-seq 81 | "Takes a HeaderIterator and returns a seq of vectors of name/value 82 | pairs of headers." 83 | [^HeaderIterator headers] 84 | (for [^Header h (iterator-seq headers)] 85 | [(.getName h) (.getValue h)])) 86 | 87 | (defn assoc-join 88 | "Like assoc, but will join multiple values into a vector if the 89 | given key is already present into the map." 90 | [headers name value] 91 | (update-in headers [name] 92 | (fn [existing] 93 | (cond (vector? existing) 94 | (conj existing value) 95 | (nil? existing) 96 | value 97 | :else 98 | [existing value])))) 99 | 100 | ;; a map implementation that stores both the original (or canonical) 101 | ;; key and value for each key/value pair, but performs lookups and 102 | ;; other operations using the normalized -- this allows a value to be 103 | ;; looked up by many similar keys, and not just the exact precise key 104 | ;; it was originally stored with. 105 | (potemkin/def-map-type HeaderMap [m mta] 106 | (get [_ k v] 107 | (second (get m (normalize k) [nil v]))) 108 | (assoc [_ k v] 109 | (HeaderMap. (assoc m (normalize k) [(if (keyword? k) 110 | (canonicalize k) 111 | k) 112 | v]) 113 | mta)) 114 | (dissoc [_ k] 115 | (HeaderMap. (dissoc m (normalize k)) mta)) 116 | (keys [_] 117 | (map first (vals m))) 118 | (meta [_] 119 | mta) 120 | (with-meta [_ mta] 121 | (HeaderMap. m mta)) 122 | 123 | clojure.lang.Associative 124 | (containsKey [_ k] 125 | (contains? m (normalize k))) 126 | (entryAt [_ k] 127 | (if (contains? m (normalize k)) 128 | (clojure.lang.MapEntry. k (get _ k)))) 129 | 130 | (empty [_] 131 | (HeaderMap. {} nil))) 132 | 133 | (defn header-map 134 | "Returns a new header map with supplied mappings." 135 | [& keyvals] 136 | (into (HeaderMap. {} nil) 137 | (apply array-map keyvals))) 138 | 139 | (defn- header-map-request 140 | [req] 141 | (let [req-headers (:headers req)] 142 | (if req-headers 143 | (-> req (assoc :headers (into (header-map) req-headers) 144 | :use-header-maps-in-response? true)) 145 | req))) 146 | 147 | (defn wrap-header-map 148 | "Middleware that converts headers from a map into a header-map." 149 | [client] 150 | (fn 151 | ([req] 152 | (client (header-map-request req))) 153 | ([req respond raise] 154 | (client (header-map-request req) respond raise)))) 155 | -------------------------------------------------------------------------------- /src/clj_http/links.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.links 2 | "Namespace dealing with HTTP link headers") 3 | 4 | (def ^:private quoted-string 5 | #"\"((?:[^\"]|\\\")*)\"") 6 | 7 | (def ^:private token 8 | #"([^,\";]*)") 9 | 10 | (def ^:private link-param 11 | (re-pattern (str "(\\w+)=(?:" quoted-string "|" token ")"))) 12 | 13 | (def ^:private uri-reference 14 | #"<([^>]*)>") 15 | 16 | (def ^:private link-value 17 | (re-pattern (str uri-reference "((?:\\s*;\\s*" link-param ")*)"))) 18 | 19 | (def ^:private link-header 20 | (re-pattern (str "(?:\\s*(" link-value ")\\s*,?\\s*)"))) 21 | 22 | (defn read-link-params [params] 23 | (into {} 24 | (for [[_ name quot tok] (re-seq link-param params)] 25 | [(keyword name) (or quot tok)]))) 26 | 27 | (defn read-link-value [value] 28 | (let [[_ uri params] (re-matches link-value value) 29 | param-map (read-link-params params)] 30 | [(keyword (:rel param-map)) 31 | (-> param-map 32 | (assoc :href uri) 33 | (dissoc :rel))])) 34 | 35 | (defn read-link-headers [header] 36 | (->> (re-seq link-header header) 37 | (map second) 38 | (map read-link-value) 39 | (into {}))) 40 | 41 | (defn- links-response 42 | [response] 43 | (if-let [link-headers (get-in response [:headers "link"])] 44 | (let [link-headers (if (coll? link-headers) 45 | link-headers 46 | [link-headers])] 47 | (assoc response 48 | :links 49 | (into {} (map read-link-headers link-headers)))) 50 | response)) 51 | 52 | (defn wrap-links 53 | "Add a :links key to the response map that contains parsed Link headers. The 54 | links will be represented as a map, with the 'rel' value being the key. The 55 | URI is placed under the 'href' key, to mimic the HTML link element. 56 | 57 | e.g. Link: ; rel=next; title=\"Page 2\" 58 | => {:links {:next {:href \"http://example.com/page2.html\" 59 | :title \"Page 2\"}}}" 60 | [client] 61 | (fn 62 | ([request] 63 | (links-response (client request))) 64 | ([request respond raise] 65 | (client request #(respond (links-response %)) raise)))) 66 | -------------------------------------------------------------------------------- /src/clj_http/multipart.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.multipart 2 | "Namespace used for clj-http to create multipart entities and bodies." 3 | (:import [java.io File InputStream] 4 | org.apache.http.Consts 5 | org.apache.http.entity.ContentType 6 | [org.apache.http.entity.mime HttpMultipartMode MultipartEntityBuilder] 7 | [org.apache.http.entity.mime.content ByteArrayBody ContentBody FileBody InputStreamBody StringBody])) 8 | 9 | ;; we don't need to make a fake byte-array every time, only once 10 | (def byte-array-type (type (byte-array 0))) 11 | 12 | (defmulti 13 | make-multipart-body 14 | "Create a body object from the given map, dispatching on the type of its 15 | content. By default supported content body types are: 16 | - String 17 | - byte array (requires providing name) 18 | - InputStream (requires providing name) 19 | - File 20 | - org.apache.http.entity.mime.content.ContentBody (which is just returned)" 21 | (fn [multipart] (type (:content multipart)))) 22 | 23 | (defmethod make-multipart-body nil 24 | [multipart] 25 | (throw (Exception. "Multipart content cannot be nil"))) 26 | 27 | (defmethod make-multipart-body :default 28 | [multipart] 29 | (throw (Exception. (str "Unsupported type for multipart content: " 30 | (type (:content multipart)))))) 31 | 32 | (defmethod make-multipart-body File 33 | ;; Create a FileBody object from the given map, requiring at least :content 34 | [{:keys [^String name ^String mime-type ^File content ^String encoding]}] 35 | (cond 36 | (and name mime-type content encoding) 37 | (FileBody. content (ContentType/create mime-type encoding) name) 38 | 39 | (and mime-type content encoding) 40 | (FileBody. content (ContentType/create mime-type encoding)) 41 | 42 | (and name mime-type content) 43 | (FileBody. content (ContentType/create mime-type) name) 44 | 45 | (and mime-type content) 46 | (FileBody. content (ContentType/create mime-type)) 47 | 48 | content 49 | (FileBody. content) 50 | 51 | :else 52 | (throw (Exception. "Multipart file body must contain at least :content")))) 53 | 54 | (defmethod make-multipart-body InputStream 55 | ;; Create an InputStreamBody object from the given map, requiring at least 56 | ;; :content and :name. If no :length is specified, clj-http will use 57 | ;; chunked transfer-encoding, if :length is specified, clj-http will 58 | ;; workaround things be proxying the InputStreamBody to return a length. 59 | [{:keys [^String name ^String mime-type ^InputStream content length]}] 60 | (cond 61 | (and content name length) 62 | (if mime-type 63 | (proxy [InputStreamBody] [content (ContentType/create mime-type) name] 64 | (getContentLength [] 65 | length)) 66 | (proxy [InputStreamBody] [content name] 67 | (getContentLength [] 68 | length))) 69 | 70 | (and content mime-type name) 71 | (InputStreamBody. content (ContentType/create mime-type) name) 72 | 73 | (and content name) 74 | (InputStreamBody. content name) 75 | 76 | :else 77 | (throw (Exception. (str "Multipart input stream body must contain " 78 | "at least :content and :name"))))) 79 | 80 | (defmethod make-multipart-body byte-array-type 81 | ;; Create a ByteArrayBody object from the given map, requiring at least 82 | ;; :content and :name. 83 | [{:keys [^String name ^String mime-type ^bytes content]}] 84 | (cond 85 | (and content name mime-type) 86 | (ByteArrayBody. content (ContentType/create mime-type) name) 87 | 88 | (and content name) 89 | (ByteArrayBody. content name) 90 | 91 | :else 92 | (throw (Exception. (str "Multipart byte array body must contain " 93 | "at least :content and :name"))))) 94 | 95 | (defmulti ^java.nio.charset.Charset encoding-to-charset class) 96 | (defmethod encoding-to-charset nil [encoding] nil) 97 | (defmethod encoding-to-charset java.nio.charset.Charset [encoding] encoding) 98 | (defmethod encoding-to-charset java.lang.String [encoding] 99 | (java.nio.charset.Charset/forName encoding)) 100 | 101 | (defmethod make-multipart-body String 102 | ;; Create a StringBody object from the given map, requiring at least :content. 103 | ;; If :encoding is specified, it will be created using the Charset for that 104 | ;; encoding. 105 | [{:keys [^String mime-type ^String content encoding]}] 106 | (cond 107 | (and content mime-type encoding) 108 | (StringBody. 109 | content (ContentType/create mime-type (encoding-to-charset encoding))) 110 | 111 | (and content encoding) 112 | (StringBody. 113 | content (ContentType/create "text/plain" (encoding-to-charset encoding))) 114 | 115 | content 116 | (StringBody. content (ContentType/create "text/plain" Consts/UTF_8)))) 117 | 118 | (defmethod make-multipart-body ContentBody 119 | ;; Use provided org.apache.http.entity.mime.content.ContentBody directly 120 | [{:keys [^ContentBody content]}] 121 | content) 122 | 123 | (defn- multipart-workaround 124 | "Workaround for AsyncHttpClient to bypass 25kb restriction on getContent. 125 | 126 | See https://github.com/dakrone/clj-http/issues/560. 127 | " 128 | [^org.apache.http.entity.mime.MultipartFormEntity mp-entity] 129 | (reify org.apache.http.HttpEntity 130 | (isRepeatable [_] (.isRepeatable mp-entity)) 131 | (isChunked [_] (.isChunked mp-entity)) 132 | (isStreaming [_] (.isStreaming mp-entity)) 133 | (getContentLength [_] (.getContentLength mp-entity)) 134 | (getContentType [_] (.getContentType mp-entity)) 135 | (getContentEncoding [_] (.getContentEncoding mp-entity)) 136 | (consumeContent [_] (.consumeContent mp-entity)) 137 | (getContent [_] 138 | (let [os (java.io.ByteArrayOutputStream.)] 139 | (.writeTo mp-entity os) 140 | (.flush os) 141 | (java.io.ByteArrayInputStream. (.toByteArray os)))) 142 | (writeTo [_ output-stream] (.writeTo mp-entity output-stream)))) 143 | 144 | (defn create-multipart-entity 145 | "Takes a multipart vector of maps and creates a MultipartEntity with each 146 | map added as a part, depending on the type of content." 147 | [multipart {:keys [mime-subtype multipart-mode multipart-charset] 148 | :or {mime-subtype "form-data" 149 | multipart-mode HttpMultipartMode/STRICT}}] 150 | (let [mp-entity (doto (MultipartEntityBuilder/create) 151 | (.setMode multipart-mode) 152 | (.setMimeSubtype mime-subtype))] 153 | (when multipart-charset 154 | (.setCharset mp-entity (encoding-to-charset multipart-charset))) 155 | (doseq [m multipart] 156 | (let [name (or (:part-name m) (:name m)) 157 | part (make-multipart-body m)] 158 | (.addPart mp-entity name part))) 159 | (multipart-workaround 160 | (.build mp-entity)))) 161 | -------------------------------------------------------------------------------- /src/clj_http/util.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.util 2 | "Helper functions for the HTTP client." 3 | (:require [clojure.string :refer [blank? lower-case split trim]] 4 | [clojure.walk :refer [postwalk]]) 5 | (:import [java.io BufferedInputStream ByteArrayInputStream ByteArrayOutputStream EOFException InputStream PushbackInputStream] 6 | [java.net URLDecoder URLEncoder] 7 | [java.util.zip DeflaterInputStream GZIPInputStream GZIPOutputStream InflaterInputStream] 8 | org.apache.commons.codec.binary.Base64 9 | org.apache.commons.io.IOUtils)) 10 | 11 | (defn utf8-bytes 12 | "Returns the encoding's bytes corresponding to the given string. If no 13 | encoding is specified, UTF-8 is used." 14 | [^String s & [^String encoding]] 15 | (.getBytes s (or encoding "UTF-8"))) 16 | 17 | (defn utf8-string 18 | "Returns the String corresponding to the given encoding's decoding of the 19 | given bytes. If no encoding is specified, UTF-8 is used." 20 | [^"[B" b & [^String encoding]] 21 | (String. b (or encoding "UTF-8"))) 22 | 23 | (defn url-decode 24 | "Returns the form-url-decoded version of the given string, using either a 25 | specified encoding or UTF-8 by default." 26 | [^String encoded & [^String encoding]] 27 | (URLDecoder/decode encoded (or encoding "UTF-8"))) 28 | 29 | (defn url-encode 30 | "Returns an UTF-8 URL encoded version of the given string." 31 | [^String unencoded & [^String encoding]] 32 | (URLEncoder/encode unencoded (or encoding "UTF-8"))) 33 | 34 | (defn base64-encode 35 | "Encode an array of bytes into a base64 encoded string." 36 | [unencoded] 37 | (utf8-string (Base64/encodeBase64 unencoded))) 38 | 39 | (defn gunzip 40 | "Returns a gunzip'd version of the given byte array." 41 | [b] 42 | (when b 43 | (cond 44 | (instance? InputStream b) 45 | (let [^PushbackInputStream b (PushbackInputStream. b) 46 | first-byte (int (try (.read b) (catch EOFException _ -1)))] 47 | (case first-byte 48 | -1 b 49 | (do (.unread b first-byte) 50 | (GZIPInputStream. b)))) 51 | :else 52 | (IOUtils/toByteArray (GZIPInputStream. (ByteArrayInputStream. b)))))) 53 | 54 | (defn gzip 55 | "Returns a gzip'd version of the given byte array." 56 | [b] 57 | (when b 58 | (let [baos (ByteArrayOutputStream.) 59 | gos (GZIPOutputStream. baos)] 60 | (IOUtils/copy (ByteArrayInputStream. b) gos) 61 | (.close gos) 62 | (.toByteArray baos)))) 63 | 64 | (defn force-stream 65 | "Force b as InputStream if it is a ByteArray." 66 | ^InputStream [b] 67 | (if (instance? InputStream b) 68 | b 69 | (ByteArrayInputStream. b))) 70 | 71 | (defn force-byte-array 72 | "force b as byte array if it is an InputStream, also close the stream" 73 | ^bytes [b] 74 | (if (instance? InputStream b) 75 | (let [^PushbackInputStream bs (PushbackInputStream. b)] 76 | (try 77 | (let [first-byte (int (try (.read bs) (catch EOFException _ -1)))] 78 | (case first-byte 79 | -1 (byte-array 0) 80 | (do (.unread bs first-byte) 81 | (IOUtils/toByteArray bs)))) 82 | (finally (.close bs)))) 83 | b)) 84 | 85 | (defn force-string 86 | "Convert s (a ByteArray or InputStream) to String." 87 | ^String [s ^String charset] 88 | (if (instance? InputStream s) 89 | (let [^PushbackInputStream bs (PushbackInputStream. s)] 90 | (try 91 | (let [first-byte (int (try (.read bs) (catch EOFException _ -1)))] 92 | (case first-byte 93 | -1 "" 94 | (do (.unread bs first-byte) 95 | (IOUtils/toString bs charset)))) 96 | (finally (.close bs)))) 97 | (IOUtils/toString ^"[B" s charset))) 98 | 99 | (defn inflate 100 | "Returns a zlib inflate'd version of the given byte array or InputStream." 101 | [b] 102 | (when b 103 | ;; This weirdness is because HTTP servers lie about what kind of deflation 104 | ;; they're using, so we try one way, then if that doesn't work, reset and 105 | ;; try the other way 106 | (let [stream (BufferedInputStream. (if (instance? InputStream b) 107 | b 108 | (ByteArrayInputStream. b))) 109 | _ (.mark stream 512) 110 | iis (InflaterInputStream. stream) 111 | readable? (try (.read iis) true 112 | (catch java.util.zip.ZipException _ false))] 113 | (.reset stream) 114 | (if readable? 115 | (InflaterInputStream. stream) 116 | (InflaterInputStream. stream (java.util.zip.Inflater. true)))))) 117 | 118 | (defn deflate 119 | "Returns a deflate'd version of the given byte array." 120 | [b] 121 | (when b 122 | (IOUtils/toByteArray (DeflaterInputStream. (ByteArrayInputStream. b))))) 123 | 124 | (defn lower-case-keys 125 | "Recursively lower-case all map keys that are strings." 126 | [m] 127 | (let [f (fn [[k v]] (if (string? k) [(lower-case k) v] [k v]))] 128 | (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) m))) 129 | 130 | (defn opt 131 | "Check the request parameters for a keyword boolean option, with or without 132 | the ? 133 | 134 | Returns false if either of the values are false, or the value of 135 | (or key1 key2) otherwise (truthy)" 136 | [req param] 137 | (let [param-? (keyword (str (name param) "?")) 138 | v1 (clojure.core/get req param) 139 | v2 (clojure.core/get req param-?)] 140 | (if (false? v1) 141 | false 142 | (if (false? v2) 143 | false 144 | (or v1 v2))))) 145 | 146 | (defn- trim-quotes [s] 147 | (when s 148 | (clojure.string/replace s #"^\s*(\"(.*)\"|(.*?))\s*$" "$2$3"))) 149 | 150 | (defn parse-content-type 151 | "Parse `s` as an RFC 2616 media type." 152 | [s] 153 | (when-let [m (re-matches #"\s*(([^/]+)/([^ ;]+))\s*(\s*;.*)?" (str s))] 154 | {:content-type (keyword (nth m 1)) 155 | :content-type-params 156 | (->> (split (str (nth m 4)) #"\s*;\s*") 157 | (remove blank?) 158 | (map #(split % #"=")) 159 | (mapcat (fn [[k v]] [(keyword (lower-case k)) (trim-quotes v)])) 160 | (apply hash-map))})) 161 | -------------------------------------------------------------------------------- /test-resources/big_array_json.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 3 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 4 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 5 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 6 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 7 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 8 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 9 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 10 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 11 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 12 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 13 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 14 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 15 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 16 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 17 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 18 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 19 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 20 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 21 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 22 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 23 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 24 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 25 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 26 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 27 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 28 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 29 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 30 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 31 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 32 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 33 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 34 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 35 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 36 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 37 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 38 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 39 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 40 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 41 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 42 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 43 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 44 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 45 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 46 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 47 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 48 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 49 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 50 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 51 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 52 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 53 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 54 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 55 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 56 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 57 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 58 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 59 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 60 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 61 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 62 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 63 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 64 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 65 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 66 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 67 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 68 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 69 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 70 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 71 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 72 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 73 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 74 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 75 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 76 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 77 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 78 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 79 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 80 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 81 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 82 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 83 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 84 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 85 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 86 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 87 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 88 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 89 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 90 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 91 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 92 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 93 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 94 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 95 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 96 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 97 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 98 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 99 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 100 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, 101 | {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]} 102 | ] 103 | -------------------------------------------------------------------------------- /test-resources/client-keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dakrone/clj-http/a9262a6beb7824a7c2621c864229ecbfa66d7257/test-resources/client-keystore -------------------------------------------------------------------------------- /test-resources/keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dakrone/clj-http/a9262a6beb7824a7c2621c864229ecbfa66d7257/test-resources/keystore -------------------------------------------------------------------------------- /test-resources/m.txt: -------------------------------------------------------------------------------- 1 | this 2 | is 3 | some 4 | file. 5 | -------------------------------------------------------------------------------- /test-resources/small.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dakrone/clj-http/a9262a6beb7824a7c2621c864229ecbfa66d7257/test-resources/small.jpg -------------------------------------------------------------------------------- /test/clj_http/test/conn_mgr_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.test.conn-mgr-test 2 | (:require [clj-http.conn-mgr :as conn-mgr] 3 | [clj-http.core :as core] 4 | [clj-http.test.core-test :refer [run-server]] 5 | [clojure.test :refer :all] 6 | [ring.adapter.jetty :as ring]) 7 | (:import java.security.KeyStore 8 | [javax.net.ssl KeyManagerFactory TrustManagerFactory] 9 | org.apache.http.impl.conn.BasicHttpClientConnectionManager)) 10 | 11 | (def client-ks "test-resources/client-keystore") 12 | (def client-ks-pass "keykey") 13 | (def secure-request {:request-method :get :uri "/" 14 | :server-port 18084 :scheme :https 15 | :keystore client-ks :keystore-pass client-ks-pass 16 | :trust-store client-ks :trust-store-pass client-ks-pass 17 | :server-name "localhost" :insecure? true}) 18 | 19 | (defn secure-handler [req] 20 | (if (nil? (:ssl-client-cert req)) 21 | {:status 403} 22 | {:status 200})) 23 | 24 | (deftest load-keystore 25 | (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey")] 26 | (is (instance? KeyStore ks)) 27 | (is (> (.size ks) 0)))) 28 | 29 | (deftest use-existing-keystore 30 | (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey") 31 | ks (conn-mgr/get-keystore ks)] 32 | (is (instance? KeyStore ks)) 33 | (is (> (.size ks) 0)))) 34 | 35 | (deftest load-keystore-with-nil-pass 36 | (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil nil)] 37 | (is (instance? KeyStore ks)))) 38 | 39 | (def array-of-trust-manager 40 | (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey") 41 | tmf (doto (TrustManagerFactory/getInstance (TrustManagerFactory/getDefaultAlgorithm)) 42 | (.init ks))] 43 | (.getTrustManagers tmf))) 44 | 45 | (def array-of-key-manager 46 | (let [ks (conn-mgr/get-keystore "test-resources/keystore" nil "keykey") 47 | tmf (doto (KeyManagerFactory/getInstance (KeyManagerFactory/getDefaultAlgorithm)) 48 | (.init ks (.toCharArray "keykey")))] 49 | (.getKeyManagers tmf))) 50 | 51 | (deftest ^:integration ssl-client-cert-get 52 | (let [server (ring/run-jetty secure-handler 53 | {:port 18083 :ssl-port 18084 54 | :ssl? true 55 | :join? false 56 | :keystore "test-resources/keystore" 57 | :key-password "keykey" 58 | :client-auth :want})] 59 | (try 60 | (let [resp (core/request {:request-method :get :uri "/get" 61 | :server-port 18084 :scheme :https 62 | :insecure? true :server-name "localhost"})] 63 | (is (= 403 (:status resp)))) 64 | (let [resp (core/request secure-request)] 65 | (is (= 200 (:status resp)))) 66 | (finally 67 | (.stop server))))) 68 | 69 | (deftest ^:integration ssl-client-cert-get-async 70 | (let [server (ring/run-jetty secure-handler 71 | {:port 18083 :ssl-port 18084 72 | :ssl? true 73 | :join? false 74 | :keystore "test-resources/keystore" 75 | :key-password "keykey" 76 | :client-auth :want})] 77 | (try 78 | (let [resp (promise) 79 | exception (promise) 80 | _ (core/request {:request-method :get :uri "/get" 81 | :server-port 18084 :scheme :https 82 | :insecure? true :server-name "localhost" 83 | :async? true} resp exception)] 84 | (is (= 403 (:status (deref resp 1000 {:status :timeout}))))) 85 | (let [resp (promise) 86 | exception (promise) 87 | _ (core/request (assoc secure-request :async? true) resp exception)] 88 | (is (= 200 (:status (deref resp 1000 {:status :timeout}))))) 89 | 90 | (testing "with reusable connection pool" 91 | (let [pool (conn-mgr/make-reusable-async-conn-manager {:timeout 10000 92 | :keystore client-ks :keystore-pass client-ks-pass 93 | :trust-store client-ks :trust-store-pass client-ks-pass 94 | :insecure? true})] 95 | (try 96 | (let [resp (promise) exception (promise) 97 | _ (core/request {:request-method :get :uri "/get" 98 | :server-port 18084 :scheme :https 99 | :server-name "localhost" 100 | :connection-manager pool :async? true} resp exception)] 101 | (is (= 200 (:status (deref resp 1000 {:status :timeout})))) 102 | (is (:body @resp)) 103 | (is (not (realized? exception)))) 104 | (finally 105 | (conn-mgr/shutdown-manager pool))))) 106 | (finally 107 | (.stop server))))) 108 | 109 | (deftest ^:integration t-closed-conn-mgr-for-as-stream 110 | (run-server) 111 | (let [shutdown? (atom false) 112 | cm (proxy [BasicHttpClientConnectionManager] [] 113 | (shutdown [] 114 | (reset! shutdown? true)))] 115 | (try 116 | (core/request {:request-method :get :uri "/timeout" 117 | :server-port 18080 :scheme :http 118 | :server-name "localhost" 119 | ;; timeouts forces an exception being thrown 120 | :socket-timeout 1 121 | :connection-timeout 1 122 | :connection-manager cm 123 | :as :stream}) 124 | (is false "request should have thrown an exception") 125 | (catch Exception e)) 126 | (is @shutdown? "Connection manager has been shutdown"))) 127 | 128 | (deftest ^:integration t-closed-conn-mgr-for-empty-body 129 | (run-server) 130 | (let [shutdown? (atom false) 131 | cm (proxy [BasicHttpClientConnectionManager] [] 132 | (shutdown [] 133 | (reset! shutdown? true))) 134 | response (core/request {:request-method :get :uri "/unmodified-resource" 135 | :server-port 18080 :scheme :http 136 | :server-name "localhost" 137 | :connection-manager cm})] 138 | (is (nil? (:body response)) "response shouldn't have body") 139 | (is (= 304 (:status response))) 140 | (is @shutdown? "connection manager should be shutdown"))) 141 | 142 | (deftest t-reusable-conn-mgrs 143 | (let [regular (conn-mgr/make-regular-conn-manager {}) 144 | regular-reusable (conn-mgr/make-reusable-conn-manager {}) 145 | async (conn-mgr/make-regular-async-conn-manager {}) 146 | async-reusable (conn-mgr/make-reusable-async-conn-manager {}) 147 | async-reuseable (conn-mgr/make-reuseable-async-conn-manager {})] 148 | (is (false? (conn-mgr/reusable? regular))) 149 | (is (true? (conn-mgr/reusable? regular-reusable))) 150 | (is (false? (conn-mgr/reusable? async))) 151 | (is (true? (conn-mgr/reusable? async-reusable))) 152 | (is (true? (conn-mgr/reusable? async-reuseable))))) 153 | -------------------------------------------------------------------------------- /test/clj_http/test/cookies_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.test.cookies-test 2 | (:require [clj-http.cookies :refer :all] 3 | [clojure.test :refer :all]) 4 | (:import [org.apache.http.impl.cookie BasicClientCookie BasicClientCookie2])) 5 | 6 | (defn refer-private [ns] 7 | (doseq [[symbol var] (ns-interns ns)] 8 | (when (:private (meta var)) 9 | (intern *ns* symbol var)))) 10 | 11 | (refer-private 'clj-http.cookies) 12 | 13 | (def session (str "ltQGXSNp7cgNeFG6rPE06qzriaI+R8W7zJKFu4UOlX4=-" 14 | "-lWgojFmZlDqSBnYJlUmwhqXL4OgBTkra5WXzi74v+nE=")) 15 | 16 | (deftest test-compact-map 17 | (are [map expected] 18 | (is (= expected (compact-map map))) 19 | {:a nil :b 2 :c 3 :d nil} 20 | {:b 2 :c 3} 21 | {:comment nil :domain "example.com" :path "/" :ports [80 8080] :value 1} 22 | {:domain "example.com" :path "/" :ports [80 8080] :value 1})) 23 | 24 | (deftest test-decode-cookie 25 | (are [set-cookie-str expected] 26 | (is (= expected (decode-cookie set-cookie-str))) 27 | nil nil 28 | "" nil 29 | "example-cookie=example-value;Path=/" 30 | ["example-cookie" 31 | {:discard true :path "/" :secure false 32 | :value "example-value" :version 0}] 33 | "example-cookie=example-value;Domain=.example.com;Path=/" 34 | ["example-cookie" 35 | {:discard true :domain "example.com" :secure false :path "/" 36 | :value "example-value" :version 0}])) 37 | 38 | (deftest test-decode-cookies-with-seq 39 | (let [cookies (decode-cookies [(str "ring-session=" session)])] 40 | (is (map? cookies)) 41 | (is (= 1 (count cookies))) 42 | (let [cookie (get cookies "ring-session")] 43 | (is (= true (:discard cookie))) 44 | (is (nil? (:domain cookie))) 45 | (is (= "/" (:path cookie))) 46 | (is (= session (:value cookie))) 47 | (is (= 0 (:version cookie)))))) 48 | 49 | (deftest test-decode-cookies-with-string 50 | (let [cookies (decode-cookies 51 | (str "ring-session=" session ";Path=/"))] 52 | (is (map? cookies)) 53 | (is (= 1 (count cookies))) 54 | (let [cookie (get cookies "ring-session")] 55 | (is (= true (:discard cookie))) 56 | (is (nil? (:domain cookie))) 57 | (is (= "/" (:path cookie))) 58 | (is (= session (:value cookie))) 59 | (is (= 0 (:version cookie)))))) 60 | 61 | (deftest test-decode-cookie-header 62 | (are [response expected] 63 | (is (= expected (decode-cookie-header response))) 64 | {:headers {"set-cookie" "a=1"}} 65 | {:cookies {"a" {:discard true :path "/" :secure false 66 | :value "1" :version 0}} :headers {}} 67 | {:headers {"set-cookie" 68 | (str "ring-session=" session ";Path=/")}} 69 | {:cookies {"ring-session" 70 | {:discard true :path "/" :secure false 71 | :value session :version 0}} :headers {}})) 72 | 73 | (deftest test-encode-cookie 74 | (are [cookie expected] 75 | (is (= expected (encode-cookie cookie))) 76 | [:a {:value "b"}] "a=b" 77 | ["a" {:value "b"}] "a=b" 78 | ["example-cookie" 79 | {:domain ".example.com" :path "/" :value "example-value"}] 80 | "example-cookie=example-value" 81 | ["ring-session" {:value session}] 82 | (str "ring-session=" session))) 83 | 84 | (deftest test-encode-cookies 85 | (are [cookie expected] 86 | (is (= expected (encode-cookies cookie))) 87 | (sorted-map :a {:value "b"} :c {:value "d"} :e {:value "f"}) 88 | "a=b;c=d;e=f" 89 | (sorted-map "a" {:value "b"} "c" {:value "d"} "e" {:value "f"}) 90 | "a=b;c=d;e=f" 91 | {"example-cookie" 92 | {:domain ".example.com" :path "/" :value "example-value"}} 93 | "example-cookie=example-value" 94 | {"example-cookie" 95 | {:domain ".example.com" :path "/" :value "example-value" 96 | :discard true :version 0}} 97 | "example-cookie=example-value" 98 | {"ring-session" {:value session}} 99 | (str "ring-session=" session))) 100 | 101 | (deftest test-encode-cookie-header 102 | (are [request expected] 103 | (is (= expected (encode-cookie-header request))) 104 | {:cookies {"a" {:value "1"}}} 105 | {:headers {"Cookie" "a=1"}} 106 | {:cookies 107 | {"example-cookie" {:domain ".example.com" :path "/" 108 | :value "example-value"}}} 109 | {:headers {"Cookie" "example-cookie=example-value"}})) 110 | 111 | (deftest test-to-basic-client-cookie-with-simple-cookie 112 | (let [cookie (to-basic-client-cookie 113 | ["ring-session" 114 | {:value session 115 | :path "/" 116 | :domain "example.com"}])] 117 | (is (= "ring-session" (.getName cookie))) 118 | (is (= session (.getValue cookie))) 119 | (is (= "/" (.getPath cookie))) 120 | (is (= "example.com" (.getDomain cookie))) 121 | (is (nil? (.getComment cookie))) 122 | (is (nil? (.getCommentURL cookie))) 123 | (is (not (.isPersistent cookie))) 124 | (is (nil? (.getExpiryDate cookie))) 125 | (is (nil? (seq (.getPorts cookie)))) 126 | (is (not (.isSecure cookie))) 127 | (is (= 0 (.getVersion cookie))))) 128 | 129 | (deftest test-to-basic-client-cookie-with-full-cookie 130 | (let [cookie (to-basic-client-cookie 131 | ["ring-session" 132 | {:value session 133 | :path "/" 134 | :domain "example.com" 135 | :comment "Example Comment" 136 | :comment-url "http://example.com/cookies" 137 | :discard true 138 | :expires (java.util.Date. (long 0)) 139 | :ports [80 8080] 140 | :secure true 141 | :version 0}])] 142 | (is (= "ring-session" (.getName cookie))) 143 | (is (= session (.getValue cookie))) 144 | (is (= "/" (.getPath cookie))) 145 | (is (= "example.com" (.getDomain cookie))) 146 | (is (= "Example Comment" (.getComment cookie))) 147 | (is (= "http://example.com/cookies" (.getCommentURL cookie))) 148 | (is (not (.isPersistent cookie))) 149 | (is (= (java.util.Date. (long 0)) (.getExpiryDate cookie))) 150 | (is (= [80 8080] (seq (.getPorts cookie)))) 151 | (is (.isSecure cookie)) 152 | (is (= 0 (.getVersion cookie))))) 153 | 154 | (deftest test-to-basic-client-cookie-with-symbol-as-name 155 | (let [cookie (to-basic-client-cookie 156 | [:ring-session {:value session :path "/" 157 | :domain "example.com"}])] 158 | (is (= "ring-session" (.getName cookie))))) 159 | 160 | (deftest test-to-cookie-with-simple-cookie 161 | (let [[name content] 162 | (to-cookie 163 | (doto (BasicClientCookie. "example-cookie" "example-value") 164 | (.setDomain "example.com") 165 | (.setPath "/")))] 166 | (is (= "example-cookie" name)) 167 | (is (nil? (:comment content))) 168 | (is (nil? (:comment-url content))) 169 | (is (:discard content)) 170 | (is (= "example.com" (:domain content))) 171 | (is (nil? (:expires content))) 172 | (is (nil? (:ports content))) 173 | (is (not (:secure content))) 174 | (is (= 0 (:version content))) 175 | (is (= "example-value" (:value content))))) 176 | 177 | (deftest test-to-cookie-with-full-cookie 178 | (let [[name content] 179 | (to-cookie 180 | (doto (BasicClientCookie2. "example-cookie" "example-value") 181 | (.setComment "Example Comment") 182 | (.setCommentURL "http://example.com/cookies") 183 | (.setDiscard true) 184 | (.setDomain "example.com") 185 | (.setExpiryDate (java.util.Date. (long 0))) 186 | (.setPath "/") 187 | (.setPorts (int-array [80 8080])) 188 | (.setSecure true) 189 | (.setVersion 1)))] 190 | (is (= "example-cookie" name)) 191 | (is (= "Example Comment" (:comment content))) 192 | (is (= "http://example.com/cookies" (:comment-url content))) 193 | (is (= true (:discard content))) 194 | (is (= "example.com" (:domain content))) 195 | (is (= (java.util.Date. (long 0)) (:expires content))) 196 | (is (= [80 8080] (:ports content))) 197 | (is (= true (:secure content))) 198 | (is (= 1 (:version content))) 199 | (is (= "example-value" (:value content))))) 200 | 201 | (deftest test-wrap-cookies 202 | (is (= {:cookies {"example-cookie" {:discard true :domain "example.com" 203 | :path "/" :value "example-value" 204 | :version 0 :secure false}} :headers {}} 205 | ((wrap-cookies 206 | (fn [request] 207 | (is (= (get (:headers request) "Cookie") "a=1;b=2")) 208 | {:headers 209 | {"set-cookie" 210 | "example-cookie=example-value;Domain=.example.com;Path=/"}})) 211 | {:cookies (sorted-map :a {:value "1"} :b {:value "2"})}))) 212 | (is (= {:headers {"set-cookie" 213 | "example-cookie=example-value;Domain=.example.com;Path=/"}} 214 | ((wrap-cookies 215 | (fn [request] 216 | (is (= (get (:headers request) "Cookie") "a=1;b=2")) 217 | {:headers 218 | {"set-cookie" 219 | "example-cookie=example-value;Domain=.example.com;Path=/"}})) 220 | {:cookies (sorted-map :a {:value "1"} :b {:value "2"}) 221 | :decode-cookies false})))) 222 | -------------------------------------------------------------------------------- /test/clj_http/test/headers_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.test.headers-test 2 | (:require [clj-http.client :as client] 3 | [clj-http.headers :refer :all] 4 | [clj-http.util :refer [lower-case-keys]] 5 | [clojure.test :refer :all]) 6 | (:import [org.eclipse.jetty.server Request Server] 7 | org.eclipse.jetty.server.handler.AbstractHandler)) 8 | 9 | (deftest test-special-case 10 | (are [expected given] 11 | (is (= expected (special-case given))) 12 | nil nil 13 | "" "" 14 | "foo" "foo" 15 | "DNT" "dnt" 16 | "P3P" "P3P" 17 | "Content-MD5" "content-md5")) 18 | 19 | (deftest test-canonicalize 20 | (are [expected given] 21 | (is (= expected (canonicalize given))) 22 | nil nil 23 | "" "" 24 | "Date" :date 25 | "Date" :DATE 26 | "Foo-Bar-Baz" :foo-bar-baz 27 | "Content-MD5" :content-md5)) 28 | 29 | (deftest test-normalize 30 | (are [expected given] 31 | (is (= expected (normalize given))) 32 | nil nil 33 | "" "" 34 | "foo" "foo") 35 | (is (= "foo" 36 | (normalize "foo") 37 | (normalize :foo) 38 | (normalize "Foo") 39 | (normalize :FOO)))) 40 | 41 | (deftest test-assoc-join 42 | (is (= {:foo "1"} (assoc-join {} :foo "1"))) 43 | (is (= {:foo "1"} (assoc-join {:foo nil} :foo "1"))) 44 | (is (= {:foo ["1" "2"]} (assoc-join {:foo "1"} :foo "2"))) 45 | (is (= {:foo ["1" "2" "3"]} (assoc-join {:foo ["1" "2"]} :foo "3")))) 46 | 47 | (deftest test-header-map 48 | (let [m (header-map :foo "bar" "baz" "quux") 49 | m2 (assoc m :ham "eggs")] 50 | (is (= "bar" 51 | (:foo m) 52 | (:FOO m) 53 | (m :foo) 54 | (m "foo") 55 | (m "FOO") 56 | (get m "foo"))) 57 | (is (= {"baz" "quux"} 58 | (dissoc m :foo) 59 | (dissoc m "foo"))) 60 | (is (= #{"Foo" "baz"} (set (keys m)))) 61 | (is (= #{"Foo" "Ham" "baz"} (set (keys m2)))) 62 | (is (= "eggs" (m2 "ham"))) 63 | (is (= "nope" (get m2 "absent" "nope"))) 64 | (is (= "baz" (:foo (merge (header-map :foo "bar") 65 | {"Foo" "baz"})))) 66 | (let [m-with-meta (with-meta m {:withmeta-test true})] 67 | (is (= (:withmeta-test (meta m-with-meta)) true))) 68 | 69 | (testing "select-keys" 70 | (are [expected keyset] (= expected (select-keys m keyset)) 71 | {"foo" "bar"} ["foo"] 72 | {"foo" "bar"} ["foo" "non-existent-key"] 73 | {"foo" "bar" "Foo" "bar" :foo "bar"} ["foo" "Foo" :foo])))) 74 | 75 | (deftest test-empty 76 | (testing "an empty header-map is a header-map" 77 | (let [m (header-map :foo :bar)] 78 | (is (= (class m) 79 | (class (empty m))))))) 80 | 81 | (defn ^Server header-server 82 | "fixture server that copies all request headers into the response as 83 | response headers" 84 | [] 85 | ;; argh, we can't use ring for this, because it lowercases headers 86 | ;; on the server side, and we explicitly want to get back the 87 | ;; headers as they are. so we'll just use jetty directly, nbd. 88 | (doto (Server. 18181) 89 | (.setHandler (proxy [AbstractHandler] [] 90 | (handle [target 91 | ^Request base-request 92 | request 93 | response] 94 | (.setHandled base-request true) 95 | (.setStatus response 200) 96 | ;; copy over request headers verbatim 97 | (doseq [n (enumeration-seq (.getHeaderNames request))] 98 | (doseq [v (enumeration-seq (.getHeaders request n))] 99 | ;; (println n v) ;; useful for debugging 100 | (.addHeader response n v))) 101 | ;; add a response header of our own in known case 102 | (.addHeader response "Echo-Server" "Says Hi!") 103 | (.. response getWriter (print "Echo!"))))) 104 | (.start))) 105 | 106 | (deftest ^:integration test-wrap-header-map 107 | (let [server (header-server)] 108 | (try 109 | (let [headers {:foo "bar" 110 | :etag "some etag" 111 | :content-md5 "some md5" 112 | :multi ["value1" "value2"] 113 | "MySpecialCase" "something"} 114 | resp (client/get "http://localhost:18181" {:headers headers}) 115 | resp-headers (:headers resp)] 116 | (testing "basic sanity checks" 117 | (is (= "Echo!" (:body resp))) 118 | (is (= "Says Hi!" (:echo-server resp-headers))) 119 | ;; was everything copied over correctly 120 | (doseq [[k v] headers] 121 | (is (= v (resp-headers k))))) 122 | (testing "foo is available as a variety of names" 123 | (is (= "bar" 124 | (:foo resp-headers) 125 | (resp-headers "foo") 126 | (resp-headers "Foo")))) 127 | (testing "header case is preserved" 128 | (let [resp-headers (into {} resp-headers)] ;; no more magic 129 | (testing "keyword request headers are canonicalized" 130 | (is (= "bar" (resp-headers "Foo"))) 131 | (is (= "some etag" (resp-headers "ETag"))) 132 | (is (= "some md5" (resp-headers "Content-MD5"))) 133 | (is (= ["value1" "value2"] (resp-headers "Multi")))) 134 | (testing "strings are as written" 135 | (is (= "something" (resp-headers "MySpecialCase"))))))) 136 | (finally 137 | (.stop server))))) 138 | 139 | (defmacro without-header-map [& body] 140 | `(client/with-middleware '~(->> client/default-middleware 141 | (list* client/wrap-lower-case-headers) 142 | (remove #(= wrap-header-map %)) 143 | (vec)) 144 | ~@body)) 145 | 146 | (deftest ^:integration test-dont-wrap-header-map 147 | (let [server (header-server)] 148 | (try 149 | (let [headers {"foo" "bar" 150 | "etag" "some etag" 151 | "content-md5" "some md5" 152 | "multi" ["value1" "value2"] 153 | "MySpecialCase" "something"} 154 | resp (without-header-map 155 | (client/get "http://localhost:18181" {:headers headers})) 156 | resp-headers (:headers resp)] 157 | (testing "basic sanity checks" 158 | (is (= "Echo!" (:body resp))) 159 | ;; was everything copied over correctly 160 | (doseq [[k v] (lower-case-keys headers)] 161 | (is (= v (resp-headers k))))) 162 | (testing "header names are all lowercase" 163 | (is (= "bar" (resp-headers "foo"))) 164 | (is (= "some etag" (resp-headers "etag"))) 165 | (is (= "some md5" (resp-headers "content-md5"))) 166 | (is (= ["value1" "value2"] (resp-headers "multi"))) 167 | (is (= "something" (resp-headers "myspecialcase"))) 168 | (is (= "Says Hi!" (resp-headers "echo-server"))))) 169 | (finally 170 | (.stop server))))) 171 | -------------------------------------------------------------------------------- /test/clj_http/test/links_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.test.links-test 2 | (:require [clj-http.links :refer :all] 3 | [clojure.test :refer :all])) 4 | 5 | (defn- link-handler [link-header] 6 | (wrap-links (constantly {:headers {"link" link-header}}))) 7 | 8 | (deftest test-wrap-links 9 | (testing "absolute link" 10 | (let [handler (link-handler "; rel=next")] 11 | (is (= (:links (handler {})) 12 | {:next {:href "http://example.com/page2.html"}})))) 13 | (testing "relative link" 14 | (let [handler (link-handler ";rel=next")] 15 | (is (= (:links (handler {})) 16 | {:next {:href "/page2.html"}})))) 17 | (testing "extra params" 18 | (let [handler (link-handler "; rel=next; title=\"Page 2\"")] 19 | (is (= (:links (handler {})) 20 | {:next {:href "/page2.html", :title "Page 2"}})))) 21 | (testing "multiple headers" 22 | (let [handler (link-handler ";rel=prev, ;rel=next,;rel=home")] 23 | (is (= (:links (handler {})) 24 | {:prev {:href "/p1"} 25 | :next {:href "/p3"} 26 | :home {:href "/"}})))) 27 | (testing "no :links key if no link headers" 28 | (let [handler (wrap-links (constantly {:headers {}})) 29 | response (handler {})] 30 | (is (not (contains? response :links)))))) 31 | 32 | (deftest t-multiple-link-headers 33 | (let [handler (link-handler ["; rel=shorturl" 34 | "; rel=icon"]) 35 | resp (handler {})] 36 | (is (= (:links resp) 37 | {:shorturl {:href "http://example.com/Zl_A"} 38 | :icon {:href "http://example.com/foo.png"}})))) 39 | -------------------------------------------------------------------------------- /test/clj_http/test/multipart_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.test.multipart-test 2 | (:require [clj-http.multipart :refer :all] 3 | [clojure.test :refer :all]) 4 | (:import [java.io ByteArrayInputStream ByteArrayOutputStream File] 5 | java.nio.charset.Charset 6 | [org.apache.http.entity.mime.content ByteArrayBody ContentBody FileBody InputStreamBody StringBody] 7 | org.apache.http.util.EntityUtils)) 8 | 9 | (defn body-str [^StringBody body] 10 | (-> body .getReader slurp)) 11 | 12 | (defn body-bytes [^ContentBody body] 13 | (let [buf (ByteArrayOutputStream.)] 14 | (.writeTo body buf) 15 | (.toByteArray buf))) 16 | 17 | (defn body-charset [^ContentBody body] 18 | (-> body .getContentType .getCharset)) 19 | 20 | (defn body-mime-type [^ContentBody body] 21 | (-> body .getContentType .getMimeType)) 22 | 23 | (defn make-input-stream [& bytes] 24 | (ByteArrayInputStream. (byte-array bytes))) 25 | 26 | (deftest test-multipart-body 27 | (testing "nil content throws exception" 28 | (is (thrown-with-msg? Exception #"Multipart content cannot be nil" 29 | (make-multipart-body {:content nil})))) 30 | 31 | (testing "unsupported content type throws exception" 32 | (is (thrown-with-msg? 33 | Exception 34 | #"Unsupported type for multipart content: class java.lang.Object" 35 | (make-multipart-body {:content (Object.)})))) 36 | 37 | (testing "ContentBody content direct usage" 38 | (let [contentBody (StringBody. "abc")] 39 | (is (identical? contentBody 40 | (make-multipart-body {:content contentBody}))))) 41 | 42 | (testing "StringBody" 43 | 44 | (testing "can create StringBody with content only" 45 | (let [body (make-multipart-body {:content "abc"})] 46 | (is (instance? StringBody body)) 47 | (is (= "abc" (body-str body))))) 48 | 49 | (testing "can create StringBody with content and encoding" 50 | (let [body (make-multipart-body {:content "abc" :encoding "ascii"})] 51 | (is (instance? StringBody body)) 52 | (is (= "abc" (body-str body))) 53 | (is (= (Charset/forName "ascii") (body-charset body))))) 54 | 55 | (testing "can create StringBody with content and mime-type and encoding" 56 | (let [body (make-multipart-body {:content "abc" 57 | :mime-type "stream-body" 58 | :encoding "ascii"})] 59 | (is (instance? StringBody body)) 60 | (is (= "abc" (body-str body))) 61 | (is (= (Charset/forName "ascii") (body-charset body))) 62 | (is (= "stream-body" (body-mime-type body)))))) 63 | 64 | (testing "ByteArrayBody" 65 | 66 | (testing "exception thrown on missing name" 67 | (is (thrown-with-msg? 68 | Exception 69 | #"Multipart byte array body must contain at least :content and :name" 70 | (make-multipart-body {:content (byte-array [0 1 2])})))) 71 | 72 | (testing "can create ByteArrayBody with name only" 73 | (let [body (make-multipart-body {:content (byte-array [0 1 2]) 74 | :name "testname"})] 75 | (is (instance? ByteArrayBody body)) 76 | (is (= "testname" (.getFilename body))) 77 | (is (= [0 1 2] (vec (body-bytes body)))))) 78 | 79 | (testing "can create ByteArrayBody with name and mime-type" 80 | (let [body (make-multipart-body {:content (byte-array [0 1 2]) 81 | :name "testname" 82 | :mime-type "byte-body"})] 83 | (is (instance? ByteArrayBody body)) 84 | (is (= "testname" (.getFilename body))) 85 | (is (= "byte-body" (body-mime-type body))) 86 | (is (= [0 1 2] (vec (body-bytes body))))))) 87 | 88 | (testing "InputStreamBody" 89 | 90 | (testing "exception thrown on missing name" 91 | (is 92 | (thrown-with-msg? 93 | Exception 94 | #"Multipart input stream body must contain at least :content and :name" 95 | (make-multipart-body 96 | {:content (ByteArrayInputStream. (byte-array [0 1 2]))})))) 97 | 98 | (testing "can create InputStreamBody with name and content" 99 | (let [input-stream (make-input-stream 1 2 3) 100 | body (make-multipart-body {:content input-stream 101 | :name "testname"})] 102 | (is (instance? InputStreamBody body)) 103 | (is (= "testname" (.getFilename body))) 104 | (is (identical? input-stream (.getInputStream body))))) 105 | 106 | (testing "can create InputStreamBody with name, content and mime-type" 107 | (let [input-stream (make-input-stream 1 2 3) 108 | body (make-multipart-body {:content input-stream 109 | :name "testname" 110 | :mime-type "input-stream-body"})] 111 | (is (instance? InputStreamBody body)) 112 | (is (= "testname" (.getFilename body))) 113 | (is (= "input-stream-body" (body-mime-type body))) 114 | (is (identical? input-stream (.getInputStream body))))) 115 | 116 | (testing 117 | "can create input InputStreamBody name, content, mime-type and length" 118 | (let [input-stream (make-input-stream 1 2 3) 119 | body (make-multipart-body {:content input-stream 120 | :name "testname" 121 | :mime-type "input-stream-body" 122 | :length 42})] 123 | (is (instance? InputStreamBody body)) 124 | (is (= "testname" (.getFilename body))) 125 | (is (= "input-stream-body" (body-mime-type body))) 126 | (is (identical? input-stream (.getInputStream body))) 127 | (is (= 42 (.getContentLength body)))))) 128 | 129 | (testing "FileBody" 130 | 131 | (testing "can create FileBody with content only" 132 | (let [test-file (File. "testfile") 133 | body (make-multipart-body {:content test-file})] 134 | (is (instance? FileBody body)) 135 | (is (= test-file (.getFile body))))) 136 | 137 | (testing "can create FileBody with content and mime-type" 138 | (let [test-file (File. "testfile") 139 | body (make-multipart-body {:content test-file 140 | :mime-type "file-body"})] 141 | (is (instance? FileBody body)) 142 | (is (= "file-body" (body-mime-type body))) 143 | (is (= test-file (.getFile body))))) 144 | 145 | (testing "can create FileBody with content, mime-type and name" 146 | (let [test-file (File. "testfile") 147 | body (make-multipart-body {:content test-file 148 | :mime-type "file-body" 149 | :name "testname"})] 150 | (is (instance? FileBody body)) 151 | (is (= "file-body" (body-mime-type body))) 152 | (is (= test-file (.getFile body))) 153 | (is (= "testname" (.getFilename body))))) 154 | 155 | (testing "can create FileBody with content and mime-type and encoding" 156 | (let [test-file (File. "testfile") 157 | body (make-multipart-body {:content test-file 158 | :mime-type "file-body" 159 | :encoding "ascii"})] 160 | (is (instance? FileBody body)) 161 | (is (= "file-body" (body-mime-type body))) 162 | (is (= (Charset/forName "ascii") (body-charset body))) 163 | (is (= test-file (.getFile body))))) 164 | 165 | (testing "can create FileBody with content, mime-type, encoding and name" 166 | (let [test-file (File. "testfile") 167 | body (make-multipart-body {:content test-file 168 | :mime-type "file-body" 169 | :encoding "ascii" 170 | :name "testname"})] 171 | (is (instance? FileBody body)) 172 | (is (= "file-body" (body-mime-type body))) 173 | (is (= (Charset/forName "ascii") (body-charset body))) 174 | (is (= test-file (.getFile body) )) 175 | (is (= "testname" (.getFilename body))))))) 176 | 177 | (deftest test-multipart-content-charset 178 | (testing "charset is nil if no multipart-charset is supplied" 179 | (let [mp-entity (create-multipart-entity [] nil)] 180 | (is (nil? (EntityUtils/getContentCharSet mp-entity))))) 181 | (testing "charset is set if a multipart-charset is supplied" 182 | (let [mp-entity (create-multipart-entity [] {:multipart-charset "UTF-8"})] 183 | (is (= "UTF-8" (EntityUtils/getContentCharSet mp-entity)))))) 184 | -------------------------------------------------------------------------------- /test/clj_http/test/util_test.clj: -------------------------------------------------------------------------------- 1 | (ns clj-http.test.util-test 2 | (:require [clj-http.util :refer :all] 3 | [clojure.java.io :as io] 4 | [clojure.test :refer :all]) 5 | (:import org.apache.commons.io.input.NullInputStream 6 | org.apache.commons.io.IOUtils)) 7 | 8 | (deftest test-lower-case-keys 9 | (are [map expected] 10 | (is (= expected (lower-case-keys map))) 11 | nil nil 12 | {} {} 13 | {"Accept" "application/json"} {"accept" "application/json"} 14 | {"X" {"Y" "Z"}} {"x" {"y" "Z"}})) 15 | 16 | (deftest t-option-retrieval 17 | (is (= (opt {:thing? true :thing true} :thing) true)) 18 | (is (= (opt {:thing? false :thing true} :thing) false)) 19 | (is (= (opt {:thing? false :thing false} :thing) false)) 20 | (is (= (opt {:thing? true :thing nil} :thing) true)) 21 | (is (= (opt {:thing? nil :thing true} :thing) true)) 22 | (is (= (opt {:thing? false :thing nil} :thing) false)) 23 | (is (= (opt {:thing? nil :thing false} :thing) false)) 24 | (is (= (opt {:thing? nil :thing nil} :thing) nil)) 25 | (is (= (opt {:thing? :a :thing nil} :thing) :a))) 26 | 27 | (deftest test-parse-content-type 28 | (are [s expected] 29 | (is (= expected (parse-content-type s))) 30 | nil nil 31 | "" nil 32 | "application/json" 33 | {:content-type :application/json 34 | :content-type-params {}} 35 | " application/json " 36 | {:content-type :application/json 37 | :content-type-params {}} 38 | "application/json; charset=UTF-8" 39 | {:content-type :application/json 40 | :content-type-params {:charset "UTF-8"}} 41 | " application/json; charset=UTF-8 " 42 | {:content-type :application/json 43 | :content-type-params {:charset "UTF-8"}} 44 | " application/json; charset=\"utf-8\" " 45 | {:content-type :application/json 46 | :content-type-params {:charset "utf-8"}} 47 | "text/html; charset=ISO-8859-4" 48 | {:content-type :text/html 49 | :content-type-params {:charset "ISO-8859-4"}} 50 | "text/html; charset=" 51 | {:content-type :text/html 52 | :content-type-params {:charset nil}})) 53 | 54 | (deftest test-force-byte-array 55 | (testing "empty InputStream returns empty byte-array" 56 | (is (= 0 (alength (force-byte-array (NullInputStream. 0)))))) 57 | (testing "InputStream contain bytes for JPEG file is coereced properly" 58 | (let [jpg-path "test-resources/small.jpg"] 59 | ;; coerce to seq to force byte-by-byte comparison 60 | (is (= (seq (IOUtils/toByteArray (io/input-stream jpg-path))) 61 | (seq (force-byte-array (io/input-stream jpg-path)))))))) 62 | 63 | (deftest test-gunzip 64 | (testing "with input streams" 65 | (testing "with empty stream, does not apply gunzip stream" 66 | (is (= "" (slurp (gunzip (force-stream (byte-array 0))))))) 67 | (testing "with non-empty stream, gunzip decompresses data" 68 | (let [data "hello world"] 69 | (is (= data 70 | (slurp (gunzip (force-stream (gzip (.getBytes data))))))))))) 71 | -------------------------------------------------------------------------------- /test/header-html5-test.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | titletext 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | This is the body 23 | 24 | 25 | -------------------------------------------------------------------------------- /test/header-test.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | titletext 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | This is the body 25 | 26 | 27 | -------------------------------------------------------------------------------- /test/jetty-logging.properties: -------------------------------------------------------------------------------- 1 | # quiet down jetty's logging 2 | org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog 3 | org.eclipse.jetty.LEVEL=WARN 4 | -------------------------------------------------------------------------------- /test/log4j2.properties: -------------------------------------------------------------------------------- 1 | status = error 2 | dest = err 3 | name = PropertiesConfig 4 | 5 | filter.threshold.type = ThresholdFilter 6 | filter.threshold.level = debug 7 | 8 | appender.console.type = Console 9 | appender.console.name = STDOUT 10 | appender.console.layout.type = PatternLayout 11 | appender.console.layout.pattern = %d | %-5p | [%t] | %c | %m%n 12 | 13 | rootLogger.level = info 14 | rootLogger.appenderRef.stdout.ref = STDOUT 15 | 16 | # Set this to debug to log all data to/from server 17 | # See https://hc.apache.org/httpcomponents-client-4.5.x/logging.html 18 | logger.wire.name = org.apache.http.wire 19 | logger.wire.level = info --------------------------------------------------------------------------------