├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── README_zh.md ├── pom.xml └── src ├── main └── java │ └── com │ └── roxstudio │ └── utils │ └── CUrl.java └── test ├── java └── com │ └── roxstudio │ └── utils │ └── CUrlTest.java └── resources ├── a2.png ├── global_sign_root_r1.jks └── random_root.jks /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | java-curl.iml 25 | .idea 26 | target 27 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | ## Version 1.3.0.0 3 | ### Feature 4 | * Add support of `--cacert` to provide custom truststore 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Licence](https://img.shields.io/badge/licence-Apache%20Licence%20%282.0%29-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) 2 | [![Maven Central](https://img.shields.io/maven-central/v/com.github.rockswang/java-curl.svg)](https://mvnrepository.com/artifact/com.github.rockswang/java-curl) 3 | [![996.icu](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu) 4 | 5 | [中文说明](README_zh.md) 6 | 7 | # Introduction 8 | java-curl is a pure-java HTTP utility implemented based on HttpURLConnection in the standard JRE, while the usage references to the commonly-used CURL command line tool under Linux. 9 | 10 | # Features 11 | * Based on the standard JRE, the source compatibility level is 1.6, can be used on Java server-side, Android and other Java environments. 12 | * The code is super compact (one java file with less than 2000 lines), without any external dependencies, can be easily reused at source level. 13 | * Easy to use, fully compatible with the most common switches of CURL tool, can be directly used as a command line tool. 14 | * Support all HTTP methods; Support multi-part file uploads; Support simple HTTP authentication. 15 | * Use ThreadLocal to solve the problem that cookies can only be stored globally in standard Java, cookies are maintained isolated for each thread. 16 | * The cookie-store in the thread can be serialized and saved, which is convenient for setting up a cookie pool. 17 | * Support HTTPS; Support self-signed certificate (JKS/BKS); Support ignoring certificate security check. 18 | * Support per-connection proxy; Support HTTP/HTTPS proxy authorization. 19 | * The redirect behavior can be controlled, and the response headers of each redirect-step can be obtained. 20 | * Support programming custom response resolver, easy to convert raw responses into JSON/HTML/XML format directly with Jackson/Gson/Jsoup/DOM4J or other 3rd-party libraries. 21 | * Support failed retry, programmable custom recoverable exceptions. 22 | 23 | # Description 24 | 25 | #### About switches and shortcuts 26 | * All switches can be passed in via `CUrl.opt(...)` method. For a list of supported switches, please refer to the [table](#supported-switches). 27 | * Some frequently used switches provide short-cut methods, please refer to the [table](#supported-switches) and source code. 28 | * The `opt()` method accepts multiple parameters and values. Note that if a CUrl switch needing a value, then the switch and value should be passed-in as two method parameters, e.g.: 29 | - `curl.opt("-d", "a=b", "-L")` 30 | - The above example gives two command line switches, namely post data "a=b" and following redirect automatically 31 | 32 | #### About CUrl.IO and its subclasses 33 | * CURL in Linux is a command line tool that is designed to read and write physical files, while java-curl as a programming library, support ByteArray or InputStream/OutputStream objects for reading and writing. 34 | * `CUrl.IO` is the abstracted interface for both input and output, its subclasses include: 35 | - `CUrl.MemIO`: corresponds to a byte buffer for direct memory access 36 | - `CUrl.FileIO`: corresponding to physical file 37 | - `CUrl.WrappedIO`: Simple wrapper for either InputStream or OutputStream 38 | * Multiple methods can use IO as a parameter, including: 39 | - `cert(io, password)`: Read the client certificate from IO 40 | - `data(io, binary)`: read POST data from IO 41 | - `form(name, io)`: Read file/text item from IO to submit an multi-part post for file-uploading 42 | - `cookie(io)`: Read cookies from IO 43 | - `cookieJar(io)`: Save cookies to IO 44 | - `dumpHeader(io)`: dumps the response header to IO 45 | - `stdout/stderr`: redirect standard-output/standard-error to IO 46 | * Following the CURL manual, "-" can be used to represent stdout, e.g.: 47 | - `curl("http://...").opt("-D", "-", "-c", "-")` 48 | - The above example initiates a request and outputs both the response header and website cookies to stdout 49 | 50 | #### About Cookies 51 | * There are two ways for handling cookies in standard Java. The first is to handle the *Cookie* request header and the *Set-Cookie* response header from the low level. Jsoup uses this approach, but there are some problems, including: 52 | - In addition to the key-value pairs, *Set-Cookie* contains domain, path, expire, httpOnly and other attributes, it's possible that multiple cookies with the same name, Jsoup use Map to store cookies, sometimes it leads to problems 53 | - According to some real-world tests, some versions of the JRE have a bug that loses the *Set-Cookie* value 54 | * The second way is to use Java's own `CookieManager/CookieStore`, but there is a serious problem, the API design is not reasonable, `CookieStore` can only have one globally singleton instance. That means in one VM, if multiple requests access the same site concurrently, then they always share the same cookies, this is not acceptable in many circumstances. 55 | * CUrl class implements a `ThreadLocal`-based `CookieStore`, each thread has a separate cookie-store, which solves the above problem perfectly. 56 | * In addition to the `--cookie/--cookie-jar` parameter, you can also use `getCookieStore` to get the `CookieStore` singleton, directly call its `add/getCookies` and other methods to read and write the current thread's cookies. 57 | * Note 1: This class is slightly different from the CURL tool for convenience of use. Subsequent requests from the same thread do not automatically clear the cookie store. Therefore, for different urls on the same website, you don't have to add the `--cookie/--cookie-jar` parameter every time. 58 | * Note 2: If you are using a thread pool, because the threads in the pool can be reused, to avoid cookie pollution, please add a `cookie("")` call on the first request in the thread, which will clear the thread-local cookie-store. 59 | 60 | #### About CUrl.Resolver and its subclasses 61 | * `CUrl.Resolver` is used to directly deserialize the raw response byte array into custom Java object, such as Xml, Json, Html, etc., can be combined with DOM4J, Jackson/Gson, Jsoup and other third-party libraries. 62 | * In the implementation of `Resolver.resolve()` method, if `CUrl.Recoverable` or its subclass instances are thrown, then this fail can be retried. If retry parameters are specified, CUrl will automatically retry the given number of times or given duration 63 | - Example: Even though the server API returns a response of status 200, but the business level error is "Please try again later". At this time, even if the request itself is successful, you can still throw a `Recoverable` to instruct CUrl to retry. 64 | 65 | #### About HTTPS 66 | * For sites with valid certificates issued by legal certification authorities, direct access is available. 67 | * You can specify a self-signed certificate (since 1.2.2) using `cert(io, password)` or `opt("-E", "path/to/file\:password")`. 68 | * You can also use `insecure()` or `opt("-k")` to instruct CUrl to ignore certificate security checks. 69 | * Currently CA certificates is not supported. If you are using a traffic capture tool to intercept HTTPS requests, please ignore the certificate security check. 70 | * You can use openssl, keytool to convert between PEM/P12/JKS/BKS certificates file format, see Example 8. 71 | 72 | #### About redirects 73 | * By default, the redirect is not automatically followed. Please use `location()` or `opt("-L")` to indicate that the redirect should be automatically followed. 74 | * Like the CURL tool, only HTTP-30X redirects are supported, does not support refresh headers, page metas, etc. 75 | * If you do not follow the redirect, you can use `CUrl.getLocations().get(0)` to get the redirected location URL after getting the 30X response. 76 | 77 | # Examples 78 | 79 | ##### Example 1：POST form submission. Two `data()` call demonstrations that `--data` switch can be specified multiple times, and the parameter values can be overwritten. 80 | ```java 81 | public void httpPost() { 82 | CUrl curl = new CUrl("http://httpbin.org/post") 83 | .data("hello=world&foo=bar") 84 | .data("foo=overwrite"); 85 | curl.exec(); 86 | assertEquals(200, curl.getHttpCode()); 87 | } 88 | ``` 89 | 90 | ##### Example 2: Accessing an HTTPS Site via Fiddler Proxy (Traffic Capture Tool) 91 | ```java 92 | public void insecureHttpsViaFiddler() { 93 | CUrl curl = new CUrl("https://httpbin.org/get") 94 | .proxy("127.0.0.1", 8888) // Use Fiddler to capture & parse HTTPS traffic 95 | .insecure(); // Ignore certificate check since it's issued by Fiddler 96 | curl.exec(); 97 | assertEquals(200, curl.getHttpCode()); 98 | } 99 | ``` 100 | 101 | ##### Example 3: Upload multiple files, one memory file, one physical file 102 | ```java 103 | public void uploadMultipleFiles() { 104 | CUrl.MemIO inMemFile = new CUrl.MemIO(); 105 | try { inMemFile.getOutputStream().write("text file content blabla...".getBytes()); } catch (Exception ignored) {} 106 | CUrl curl = new CUrl("http://httpbin.org/post") 107 | .form("formItem", "value") // a plain form item 108 | .form("file", inMemFile) // in-memory "file" 109 | .form("image", new CUrl.FileIO("D:\\tmp\\a2.png")); // A file in storage 110 | curl.exec(); 111 | assertEquals(200, curl.getHttpCode()); 112 | } 113 | ``` 114 | 115 | ##### Example 4: Simulate an AJAX request from a mobile browser and add a custom request header. Specify single request header with `header()`, or multiple request headers at a time with `headers()`. 116 | ```java 117 | public void customUserAgentAndHeaders() { 118 | String mobileUserAgent = "Mozilla/5.0 (Linux; U; Android 8.0.0; zh-cn; KNT-AL10 Build/HUAWEIKNT-AL10) " 119 | + "AppleWebKit/537.36 (KHTML, like Gecko) MQQBrowser/7.3 Chrome/37.0.0.0 Mobile Safari/537.36"; 120 | Map fakeAjaxHeaders = new HashMap(); 121 | fakeAjaxHeaders.put("X-Requested-With", "XMLHttpRequest"); 122 | fakeAjaxHeaders.put("Referer", "http://somesite.com/fake_referer"); 123 | CUrl curl = new CUrl("http://httpbin.org/get") 124 | .opt("-A", mobileUserAgent) // simulate a mobile browser 125 | .headers(fakeAjaxHeaders) // simulate an AJAX request 126 | .header("X-Auth-Token: xxxxxxx"); // other custom header, this might be calculated elsewhere 127 | curl.exec(); 128 | assertEquals(200, curl.getHttpCode()); 129 | } 130 | ``` 131 | 132 | ##### Example 5: Multi-threaded concurrent requests, inter-threaded cookies are isolated between each other 133 | ```java 134 | public void threadSafeCookies() { 135 | final CountDownLatch count = new CountDownLatch(3); 136 | final CUrl[] curls = new CUrl[3]; 137 | for (int i = 3; --i >= 0;) { 138 | final int idx = i; 139 | new Thread() { 140 | public void run() { 141 | CUrl curl = curls[idx] = new CUrl("http://httpbin.org/get") 142 | .cookie("thread" + idx + "=#" + idx); 143 | curl.exec(); 144 | count.countDown(); 145 | } 146 | }.start(); 147 | } 148 | try { count.await(); } catch (Exception ignored) {} // make sure all requests are done 149 | assertEquals(200, curls[0].getHttpCode()); 150 | assertEquals("thread0=#0", deepGet(curls[0].getStdout(jsonResolver, null), "headers.Cookie")); 151 | assertEquals("thread1=#1", deepGet(curls[1].getStdout(jsonResolver, null), "headers.Cookie")); 152 | assertEquals("thread2=#2", deepGet(curls[2].getStdout(jsonResolver, null), "headers.Cookie")); 153 | } 154 | ``` 155 | 156 | ##### Example 6: Programming a custom response resolver that convert raw response to HTML with Jsoup 157 | ```java 158 | private CUrl.Resolver htmlResolver = new CUrl.Resolver() { 159 | @SuppressWarnings("unchecked") 160 | @Override 161 | public Document resolve(int httpCode, byte[] responseBody) throws Throwable { 162 | String html = new String(responseBody, "UTF-8"); 163 | return Jsoup.parse(html); 164 | } 165 | }; 166 | 167 | public void customResolver() { 168 | CUrl curl = new CUrl("http://httpbin.org/html"); 169 | Document html = curl.exec(htmlResolver, null); 170 | assertEquals(200, curl.getHttpCode()); 171 | assertEquals("Herman Melville - Moby-Dick", html.select("h1:first-child").text()); 172 | } 173 | ``` 174 | 175 | ##### Example 7: As a command line tool, same request with Example 4 176 | ```shell 177 | java -jar java-curl-1.2.2.jar https://httpbin.org/get ^ 178 | -x 127.0.0.1:8888 -k ^ 179 | -A "Mozilla/5.0 (Linux; U; Android 8.0.0; zh-cn; KNT-AL10 Build/HUAWEIKNT-AL10) AppleWebKit/537.36 (KHTML, like Gecko) MQQBrowser/7.3 Chrome/37.0.0.0 Mobile Safari/537.36" ^ 180 | -H "Referer: http://somesite.com/fake_referer" ^ 181 | -H "X-Requested-With: XMLHttpRequest" ^ 182 | -H "X-Auth-Token: xxxxxxx" 183 | ``` 184 | 185 | ##### Example 8: Using a self-signed certificate. Uses JKS in stadard JRE, uses BKS in Android 186 | ```shell 187 | # Convert website certificate and private key to p12/pfx certificate 188 | openssl pkcs12 -export -in cert.pem -inkey key.pem -name cert -out cert.p12 189 | # Convert p12/pfx to jks format and set the password to 123456 190 | keytool -importkeystore -srckeystore cert.p12 -srcstoretype pkcs12 -srcstorepass 123456 -destkeystore cert.jks -deststorepass 123456 191 | # Convert Jks format to bks format, BKS certificate is applicable to Android platform 192 | keytool -importkeystore -srckeystore cert.jks -srcstoretype JKS -srcstorepass 123456 -destkeystore cert.bks -deststoretype BKS -deststorepass 123456 -provider org.bouncycastle.jce.provider.BouncyCastleProvider -providerpath "path/to/bcprov-jdk16-140.jar" 193 | # Call java-curl on the command line, you must specify the password of the JKS file. 194 | java -jar java-curl-1.2.2.jar https://mysecuritysite.com -E cert.jks:123456 195 | ``` 196 | 197 | # Supported Switches 198 | | Switch Name | Short-cut Method | Description | 199 | |--------------------------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 200 | | -E, --cert | cert | <certificate:password> Specify the client certificate file and password, only support JKS format certificate (only BKS is supported on Android) | 201 | | --compressed | NO | Request to gzip compressed response data (but need server side support) | 202 | | --connect-timeout | timeout | Connection timeout time, in seconds, default 0, that is, never timeout | 203 | | -b, --cookie | cookie | Read cookies from file / IO object / parameter string | 204 | | -c, --cookie-jar | cookieJar | Cookie output to file / IO object | 205 | | -d, --data, --data-ascii | data | Add post data, if used multiple times, use '&' to connect, the added form item key-value pair will overwrite the previous one.
If data starts with '@', the latter part is used as the file name, and the data is used by File read in, and delete carriage return in the file | 206 | | --data-raw | NO | Same as "--data", but not special handling for '@' | 207 | | --data-binary | NO | Same as "--data", but does not delete carriage return line feed characters when reading in files | 208 | | --data-urlencode | data(data,charset) | Same as "--data", but with Url-Encode for data, you can append a character set after this option, such as "--data-urlencode-GBK".
If the first character of the parameter value is '=': the following string is whole Url-Encode;
If the parameter value contains '=': split the string into key-value pairs separated by '&', the key-value pairs are split with '=', for key-value pairs All values in the value are Url-Encode.
If the parameter value does not contain '=':
-- If the string does not contain '@', then the entire string is Url-Encode
-- If the string contains '@' then split the string with '@', after '@' is the input file name, then read the text from the file and perform Url-Encode, the front part of '@' is the key
-- If '@' is the first character, the text read in the file is generally Url-Encode | 209 | | -D, --dump-header | dumpHeader | Output the response header of the last step jump to the given file / IO object | 210 | | -F, --form | form | Initiate file upload, add a file or form item.
- If the initial value of the parameter value is '@' or '<', the data is read from the specified file for uploading. The difference between '@' and '<' is that the file content of '@' is uploaded as a file attachment, and the file content of '<' is used as the value of the normal form item.
- Otherwise, the parameter value is used as the value of the normal form item. | 211 | | --form-string | form(formString) | Initiate file upload, add 1 non-file form item, note that this method does not specialize for '@' | 212 | | -G, --get | NO | Force the GET method. Will add the key-value pair specified by -d to the url as the query parameter | 213 | | -H, --header | header | Add a request header line with the syntax:
-- "Host: baidu.com": Add/set a normal request header key-value pair
-- "Accept:": Delete the given request header
-- "X-Custom-Header;": Add/set a custom request header with a value of null | 214 | | -I, --head | NO | Request using the HEAD method | 215 | | -k, --insecure | insecure | Ignore HTTPS certificate security check | 216 | | -v, --verbose | verbose | More verbose output with timestamp | 217 | | -L, --location | location | Automatic follow redirect (not enabled by default) | 218 | | -m, --max-time | timeout | Transmission timeout, in seconds, default 0, that is, never timeout | 219 | | -o, --output | output | Specify the output file / IO object, the default stdout, which is "-" | 220 | | -x, --proxy | proxy | Set proxy server | 221 | | -U, --proxy-user | NO | Set proxy server authorization | 222 | | -e, --referer | NO | Set the Referer request header content | 223 | | --retry | retry | Set the number of retries, default 0 | 224 | | --retry-delay | retry | Set the delay between two retries, in seconds, default 0 | 225 | | --retry-max-time | retry | Set the maximum retry total time, in seconds, default 0, that is, never time out | 226 | | -s, --silent | NO | Set silent mode, which suppress all outputs | 227 | | --stderr | stderr | Set stderr output file / IO object, default stdout | 228 | | -u, --user | NO | Set the HTTP Authorization information. Note that it is only used for simple HTTP authentication, which is the case where the system dialog box pops up in the browser. | 229 | | --url | CUrl, url | Set the request address, this CUrl library does not support multiple url requests. | 230 | | -A, --user-agent | NO | Set the "User-Agent" request header content | 231 | | -X, --request | NO | Specify HTTP request method | 232 | | --x-max-download | NO | Abandon download after the transfer reaches a given number of bytes (inaccurate) | 233 | | --x-tags | NO | Set additional key-value pairs to be stored in the current CUrl instance for passing additional parameters in programming | 234 | 235 | ### Contribute 236 | Please increment version number, following [semvar](https://semver.org/) 237 | 238 | #### Testing 239 | Kindly ensure testing coverage, you need a forward proxy on `127.0.0.1:8888` for the testing to pass. -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- 1 | [![Licence](https://img.shields.io/badge/licence-Apache%20Licence%20%282.0%29-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) 2 | [![Maven Central](https://img.shields.io/maven-central/v/com.github.rockswang/java-curl.svg)](https://mvnrepository.com/artifact/com.github.rockswang/java-curl) 3 | [![996.icu](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu) 4 | 5 | [English Version](README.md) 6 | 7 | # 简介 8 | CUrl类是以Linux下常用命令行工具CUrl为参考，基于标准Java运行库中的HttpURLConnection实现的Http工具类。 9 | 10 | # 特点 11 | * 基于标准Java运行库的Http类实现，源码兼容级别为1.6，适用性广泛，可用于服务端、Android等Java环境 12 | * 代码精简紧凑，仅一个1000余行的Java源文件，无任何外部依赖，可不用Maven直接源码级重用 13 | * 简单易用，完全兼容CUrl命令行工具的常用开关，可直接作为命令行工具替代之 14 | * 支持所有HTTP Method，支持multipart多文件上传；支持简单HTTP认证 15 | * 通过ThreadLocal解决了标准Java中Cookie只能全局保存的问题，可每线程独立维护Cookie 16 | * 可将线程中保存的Cookies序列化保存，方便建立Cookies池 17 | * 支持HTTPS，支持自签名证书（JKS/BKS）；亦可忽略证书安全检查 18 | * 支持每连接代理，支持需认证的HTTP/HTTPS代理 19 | * 跳转行为可控制，可获取到每步跳转的应答头信息 20 | * 支持编程自定义应答解析器，结合Jackson/Gson/Jsoup等库即可解析JSON/HTML等格式的应答 21 | * 支持失败重试，可编程自定义可重试异常 22 | 23 | # 说明 24 | 25 | #### 关于参数和快捷方法 26 | * 所有参数均可以通过`CUrl.opt(...)`方法传入，具体支持的参数列表请参见文末表格 27 | * 部分常用参数提供了快捷方法，具体请见文末表格和源码 28 | * `opt`方法接受多个参数和值。注意，如果一个CUrl参数需要提供值，那么应该分成两个方法参数传入，比如： 29 | * `curl.opt("-d", "a=b", "-L")` 30 | * 上面例子中提供了两个命令行参数，即post数据"a=b"，以及自动跟随重定向 31 | 32 | #### 关于`CUrl.IO`及其子类 33 | * Linux中的CURL是个命令行工具，只能读取、输出物理文件，但作为编程库的java-curl，支持以字节数组或输入输出流对象作为读取和写入的目标 34 | * `CUrl.IO`即抽象出来的输入输出接口，其子类包括： 35 | * `CUrl.MemIO`：对应于ByteArrayInputStream/ByteArrayOutputStream，用于直接内存读取或写入 36 | * `CUrl.FileIO`: 对应物理文件 37 | * `CUrl.WrappedIO`: 流对象的简单包装，要么只能作为输入，要么只能作为输出 38 | * CUrl的多个方法都可以使用IO作为参数，包括： 39 | * `cert(io, password)`: 从IO读取客户端证书 40 | * `data(io, binary)`: 从IO中读取POST数据 41 | * `form(name, io)`: 从IO中读取，添加一个文件上传表单项 42 | * `cookie(io`): 从IO中读取Cookies 43 | * `cookieJar(io)`: 把Cookies保存到IO 44 | * `dumpHeader(io)`: 将应答头倾印到IO 45 | * `stdout/stderr`: 重定向标准输出/标准错误输出到IO 46 | * 注意，所有输出类参数，均可以用"-"代表stdout，比如 47 | * `curl("http://...").opt("-D", "-", "-c", "-")` 48 | * 上例发起请求并把应答头、网站Cookie均输出到stdout 49 | 50 | #### 关于Cookies 51 | * 基于标准Java处理cookies有两种方案，一种是自行处理Cookie请求头和Set-Cookie应答头，Jsoup即使用此方案，但有一些问题，包括： 52 | * Set-Cookie中除了键值对外，还有domain, path, expire, httpOnly等属性，有可能出现同名的Cookie，Jsoup用Map简单处理，有时会有问题 53 | * 据实际测试，部分版本JRE有丢失Set-Cookie值的BUG 54 | * 第二种方案即使用Java自己的CookieManager/CookieStore，但有一个严重问题，此处API设计不合理，CookieStore只能有一个全局默认单例，也就是说，一个JVM进程中如果多个请求并发访问同一站点，那么它们是共用同一份Cookie的，这在很多情况下并不适用！ 55 | * CUrl类实现了一个基于ThreadLocal的CookieStore，每条线程有独立的cookie，完美解决了上述问题 56 | * 除了`--cookie/--cookie-jar`参数外，还可以使用getCookieStore获取到CookieStore单例，直接调用其`add/getCookies`等方法读写当前线程的cookies 57 | * 注意1：本类为了方便使用，和CURL工具略有区别，同一线程的多次请求不会自动清除cookie存储。因此，对同一网站的不同url，不必每次添加`--cookie/--cookie-jar`参数 58 | * 注意2：如果使用线程池，由于池中线程会被重用，为了避免Cookie污染，请在线程中第一次请求上添加`--cookie("")`调用，这会清除本线程cookie存储 59 | 60 | #### 关于`CUrl.Resolver`及其子类 61 | * `CUrl.Resolver`用于直接将原始应答字节数组反序列化为自定义Java对象，比如Xml, Json, Html等，可以结合JDOM, Jackson/Gson, Jsoup等第三方库使用 62 | * 在`Resolver.resolve`的实现方法中，如果抛出`CUrl.Recoverable`或其子类实例，则表示此错误可重试，如果指定了重试参数，则CUrl会自动重试给定次数或给定时间 63 | * 举例：服务端API返回200的正常应答，但业务级错误为“请稍候重试”，此时即使请求本身是成功的，仍然可以抛出一个`Recoverable`异常指示CUrl重试 64 | 65 | #### 关于HTTPS 66 | * 对于有合法认证机构签发的有效证书的站点，可以直接访问 67 | * 可以使用cert(io, password)或opt("-E", "path/to/file:password")指定自签名证书 (since 1.2.2) 68 | * 也可使用insecure()/opt("-k")指示CUrl忽略证书安全检查 69 | * 不支持指定CA证书，如使用抓包工具拦截HTTPS请求，请忽略证书安全检查 70 | * 可以用openssl, keytool在PEM/P12/JKS/BKS等格式证书间相互转换，请参见例8 71 | 72 | #### 关于重定向 73 | * 默认不自动跟随重定向，请使用`location()/opt("-L")`指示自动跟随重定向 74 | * 跟CURL工具一样，只支持30X重定向，不支持refresh头部，页面meta等重定向方式 75 | * 如果不跟随重定向，可以在获取到30X应答后，使用`CUrl.getLocations().get(0)`获取到重定向的目标URL 76 | 77 | # 例子 78 | 79 | ##### 例1：POST表单提交。两次data调用演示可多次指定`--data`命令行参数，且参数值可覆盖 80 | ```java 81 | public void httpPost() { 82 | CUrl curl = new CUrl("http://httpbin.org/post") 83 | .data("hello=world&foo=bar") 84 | .data("foo=overwrite"); 85 | curl.exec(); 86 | assertEquals(200, curl.getHttpCode()); 87 | } 88 | ``` 89 | 90 | ##### 例2：通过Fiddler代理（抓包工具）访问HTTPS站点 91 | ```java 92 | public void insecureHttpsViaFiddler() { 93 | CUrl curl = new CUrl("https://httpbin.org/get") 94 | .proxy("127.0.0.1", 8888) // Use Fiddler to capture & parse HTTPS traffic 95 | .insecure(); // Ignore certificate check since it's issued by Fiddler 96 | curl.exec(); 97 | assertEquals(200, curl.getHttpCode()); 98 | } 99 | ``` 100 | 101 | ##### 例3：上传多个文件，一个内存文件，一个物理文件 102 | ```java 103 | public void uploadMultipleFiles() { 104 | CUrl.MemIO inMemFile = new CUrl.MemIO(); 105 | try { inMemFile.getOutputStream().write("text file content blabla...".getBytes()); } catch (Exception ignored) {} 106 | CUrl curl = new CUrl("http://httpbin.org/post") 107 | .form("formItem", "value") // a plain form item 108 | .form("file", inMemFile) // in-memory "file" 109 | .form("image", new CUrl.FileIO("D:\\tmp\\a2.png")); // A file in storage 110 | curl.exec(); 111 | assertEquals(200, curl.getHttpCode()); 112 | } 113 | ``` 114 | 115 | ##### 例4：模拟手机浏览器上的AJAX请求，添加自定义请求头。可用`header()`指定单一请求头，也可用`headers()`一次指定多个请求头 116 | ```java 117 | public void customUserAgentAndHeaders() { 118 | String mobileUserAgent = "Mozilla/5.0 (Linux; U; Android 8.0.0; zh-cn; KNT-AL10 Build/HUAWEIKNT-AL10) " 119 | + "AppleWebKit/537.36 (KHTML, like Gecko) MQQBrowser/7.3 Chrome/37.0.0.0 Mobile Safari/537.36"; 120 | Map fakeAjaxHeaders = new HashMap(); 121 | fakeAjaxHeaders.put("X-Requested-With", "XMLHttpRequest"); 122 | fakeAjaxHeaders.put("Referer", "http://somesite.com/fake_referer"); 123 | CUrl curl = new CUrl("http://httpbin.org/get") 124 | .opt("-A", mobileUserAgent) // simulate a mobile browser 125 | .headers(fakeAjaxHeaders) // simulate an AJAX request 126 | .header("X-Auth-Token: xxxxxxx"); // other custom header, this might be calculated elsewhere 127 | curl.exec(); 128 | assertEquals(200, curl.getHttpCode()); 129 | } 130 | ``` 131 | 132 | ##### 例5：多线程并发请求，线程间Cookies相互独立 133 | ```java 134 | public void threadSafeCookies() { 135 | final CountDownLatch count = new CountDownLatch(3); 136 | final CUrl[] curls = new CUrl[3]; 137 | for (int i = 3; --i >= 0;) { 138 | final int idx = i; 139 | new Thread() { 140 | public void run() { 141 | CUrl curl = curls[idx] = new CUrl("http://httpbin.org/get") 142 | .cookie("thread" + idx + "=#" + idx); 143 | curl.exec(); 144 | count.countDown(); 145 | } 146 | }.start(); 147 | } 148 | try { count.await(); } catch (Exception ignored) {} // make sure all requests are done 149 | assertEquals(200, curls[0].getHttpCode()); 150 | assertEquals("thread0=#0", deepGet(curls[0].getStdout(jsonResolver, null), "headers.Cookie")); 151 | assertEquals("thread1=#1", deepGet(curls[1].getStdout(jsonResolver, null), "headers.Cookie")); 152 | assertEquals("thread2=#2", deepGet(curls[2].getStdout(jsonResolver, null), "headers.Cookie")); 153 | } 154 | ``` 155 | 156 | ##### 例6：编程自定义应答解析器，使用Jsoup解析HTML 157 | ```java 158 | private CUrl.Resolver htmlResolver = new CUrl.Resolver() { 159 | @SuppressWarnings("unchecked") 160 | @Override 161 | public Document resolve(int httpCode, byte[] responseBody) throws Throwable { 162 | String html = new String(responseBody, "UTF-8"); 163 | return Jsoup.parse(html); 164 | } 165 | }; 166 | 167 | public void customResolver() { 168 | CUrl curl = new CUrl("http://httpbin.org/html"); 169 | Document html = curl.exec(htmlResolver, null); 170 | assertEquals(200, curl.getHttpCode()); 171 | assertEquals("Herman Melville - Moby-Dick", html.select("h1:first-child").text()); 172 | } 173 | ``` 174 | 175 | ##### 例7：作为命令行工具使用，请求内容参考例4 176 | ```shell 177 | java -jar java-curl-1.2.2.jar https://httpbin.org/get ^ 178 | -x 127.0.0.1:8888 -k ^ 179 | -A "Mozilla/5.0 (Linux; U; Android 8.0.0; zh-cn; KNT-AL10 Build/HUAWEIKNT-AL10) AppleWebKit/537.36 (KHTML, like Gecko) MQQBrowser/7.3 Chrome/37.0.0.0 Mobile Safari/537.36" ^ 180 | -H "Referer: http://somesite.com/fake_referer" ^ 181 | -H "X-Requested-With: XMLHttpRequest" ^ 182 | -H "X-Auth-Token: xxxxxxx" 183 | ``` 184 | 185 | ##### 例8：使用自签名证书。java平台用JKS格式证书，android平台需使用BKS格式证书 186 | ```shell 187 | # 网站证书和私钥转换成p12/pfx证书 188 | openssl pkcs12 -export -in cert.pem -inkey key.pem -name cert -out cert.p12 189 | # p12/pfx转成jks格式，并设定密码为123456 190 | keytool -importkeystore -srckeystore cert.p12 -srcstoretype pkcs12 -srcstorepass 123456 -destkeystore cert.jks -deststorepass 123456 191 | # jks格式转成bks格式，BKS证书适用于Android平台 192 | keytool -importkeystore -srckeystore cert.jks -srcstoretype JKS -srcstorepass 123456 -destkeystore cert.bks -deststoretype BKS -deststorepass 123456 -provider org.bouncycastle.jce.provider.BouncyCastleProvider -providerpath "path/to/bcprov-jdk16-140.jar" 193 | # 命令行调用java-curl，必须指定JKS文件的密码 194 | java -jar java-curl-1.2.2.jar https://mysecuritysite.com -E cert.jks:123456 195 | ``` 196 | 197 | # 支持的参数 198 | | 参数名 | 快捷方法 | 说明 | 199 | | ----------------- | --------------------- | ---- | 200 | | -E, --cert | cert | <certificate:password> 指定客户端证书文件和密码，仅支持JKS格式证书(Android上只支持BKS) | 201 | | --compressed | 无 | 请求以gzip压缩应答数据（但需服务器端支持） | 202 | | --connect-timeout | timeout | 连接超时时间，单位秒，默认0，即永不超时 | 203 | | -b, --cookie | cookie | 从文件/IO对象/参数字符串中读取Cookie | 204 | | -c, --cookie-jar | cookieJar | Cookie输出到文件/IO对象 | 205 | | -d, --data, --data-ascii | data | 添加post数据，如果多次使用，则使用'&'连接，后添加的表单项键值对会覆盖之前的
如果data以'@'开头，则后面部分作为文件名，数据由该文件读入，且删除文件中的回车换行 | 206 | | --data-raw | 无 | 同"--data"，但不对'@'特殊处理 | 207 | | --data-binary | 无 | 同"--data"，但读入文件时不删除回车换行字符 | 208 | | --data-urlencode | data(data,charset) | 同"--data"，但对数据进行Url-Encode，可以在此选项后面附加字符集，比如"--data-urlencode-GBK"
如果参数值首字符为'='：对'='后面的字符串整体进行Url-Encode* 如果参数值中包含'='：将字符串拆分为以'&'分割的键值对，键值对用'='分割，对键值对中所有的值进行Url-Encode
如果参数值中不包含'='：
--如果字符串中不包含'@'，则对字符串整体进行Url-Encode
--如果字符串中包含'@'则以'@'分割字符串，'@'后面为输入文件名，则从该文件中读取文本并进行Url-Encode，'@'前面部分为键
--如'@'为第一个字符，则文件中读出的文本整体进行Url-Encode | 209 | | -D, --dump-header | dumpHeader | 输出最后一步跳转的应答头到给定的文件/IO对象 | 210 | | -F, --form | form | 发起文件上传，添加一个文件或表单项
-如参数值首字母为'@'或'<'则从指定的文件读取数据进行上传。'@'和'<'的区别在于，'@'的文件内容作为文件附件上传，'<'的文件内容作为普通表单项的值
-否则参数值作为普通表单项的值 | 211 | | --form-string | form(formString) | 发起文件上传，添加1个非文件表单项，注意此方法不对'@'进行特殊处理 | 212 | | -G, --get | 无 | 强制使用GET方法。会把-d指定的键值对添加到url后作为查询参数 | 213 | | -H, --header | header | 添加一个请求头行，语法为：
-"Host: baidu.com": 添加/设定一行普通请求头键值对
-"Accept:": 删除给定请求头
-"X-Custom-Header;": 添加/设定一个值为空的自定义请求头 | 214 | | -I, --head | 无 | 使用HEAD方法请求 | 215 | | -k, --insecure | insecure | 忽略HTTPS证书安全检查 | 216 | | -L, --location | location | 自动跟随跳转（默认不开启） | 217 | | -m, --max-time | timeout | 传输超时时间，单位秒，默认0，即永不超时 | 218 | | -o, --output | output | 指定输出文件/IO对象，默认stdout，即"-" | 219 | | -x, --proxy | proxy | 设定代理服务器 | 220 | | -U, --proxy-user | 无 | 设定代理服务器登录信息 | 221 | | -e, --referer | 无 | 设定Referer请求头内容 | 222 | | --retry | retry | 设定重试次数，默认0 | 223 | | --retry-delay | retry | 设定两次重试之间的延迟，单位秒，默认0 | 224 | | --retry-max-time | retry | 设定最长重试总时间，单位秒，默认0，即永不超时 | 225 | | -s, --silent | 无 | 设定静默模式，即屏蔽所有输出 | 226 | | --stderr | stderr | 设定stderr的输出文件/IO对象，默认stdout | 227 | | -u, --user | 无 | 设定服务器登录信息。注意只用于简单HTTP认证，即浏览器中弹出系统对话框的情况 | 228 | | --url | CUrl, url | 设定请求地址，本CUrl库不支持多url请求 | 229 | | -A, --user-agent | 无 | 设定"User-Agent"请求头内容 | 230 | | -X, --request | 无 | 指定HTTP请求方法 | 231 | | --x-max-download | 无 | 传输达到给定字节数（非精确）后放弃下载 | 232 | | --x-tags | 无 | 设定额外的键值对信息，存储在当前CUrl实例中，用于在编程中传递额外参数 | 233 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 4.0.0 7 | 8 | com.github.rockswang 9 | java-curl 10 | ${release-version} 11 | jar 12 | 13 | 14 | org.sonatype.oss 15 | oss-parent 16 | 7 17 | 18 | 19 | java-curl 20 | Ultra-lightweight CURL implementation in pure java 1.6 21 | https://github.com/rockswang/java-curl 22 | 23 | 24 | 25 | 1.3.0.0 26 | 3.1.1 27 | 28 | 29 | 30 | 31 | junit 32 | junit 33 | 4.13.2 34 | test 35 | 36 | 37 | com.fasterxml.jackson.core 38 | jackson-databind 39 | 2.15.1 40 | test 41 | 42 | 43 | org.jsoup 44 | jsoup 45 | 1.15.4 46 | test 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-compiler-plugin 56 | 3.1 57 | 58 | 8 59 | 8 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-source-plugin 66 | 3.2.1 67 | 68 | 69 | package 70 | 71 | jar-no-fork 72 | 73 | 74 | 75 | 76 | 77 | 78 | org.apache.maven.plugins 79 | maven-javadoc-plugin 80 | 3.3.0 81 | 82 | -Xdoclint:none 83 | 84 | 85 | 86 | package 87 | 88 | jar 89 | 90 | 91 | 92 | 93 | 94 | 95 | org.apache.maven.plugins 96 | maven-gpg-plugin 97 | 1.6.0 98 | 99 | 100 | sign-artifacts 101 | verify 102 | 103 | sign 104 | 105 | 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-shade-plugin 112 | 3.3.0 113 | 114 | 115 | package 116 | 117 | shade 118 | 119 | 120 | 121 | 122 | 123 | com.roxstudio.utils.CUrl 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | The Apache Software License, Version 2.0 137 | http://www.apache.org/licenses/LICENSE-2.0.txt 138 | repo,manual 139 | 140 | 141 | 142 | 143 | 144 | Rocks Wang 145 | rockswang@foxmail.com 146 | roxstudio 147 | https://github.com/rockswang 148 | 149 | 150 | 151 | 152 | scm:git:https://github.com/rockswang/java-curl.git 153 | scm:git:https://github.com/rockswang/java-curl.git 154 | https://github.com/rockswang/java-curl 155 | ${release-version} 156 | 157 | 158 | 159 | 160 | ossrh 161 | OSS Snapshots Repository 162 | https://oss.sonatype.org/content/repositories/snapshots/ 163 | 164 | 165 | ossrh 166 | OSS Staging Repository 167 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 168 | 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /src/main/java/com/roxstudio/utils/CUrl.java: -------------------------------------------------------------------------------- 1 | package com.roxstudio.utils; 2 | 3 | import javax.net.ssl.*; 4 | import java.io.*; 5 | import java.lang.reflect.*; 6 | import java.net.*; 7 | import java.net.Proxy; 8 | import java.security.KeyStore; 9 | import java.security.KeyStoreException; 10 | import java.security.NoSuchAlgorithmException; 11 | import java.security.SecureRandom; 12 | import java.security.cert.CertificateException; 13 | import java.security.cert.X509Certificate; 14 | import java.util.*; 15 | import java.util.regex.Pattern; 16 | import java.util.zip.GZIPInputStream; 17 | import java.util.zip.InflaterInputStream; 18 | import java.text.SimpleDateFormat; 19 | import java.text.DateFormat; 20 | 21 | /** 22 | * Note: 23 | * * In order to set Restricted Headers i.e. "Origin" etc., you may need to add "-Dsun.net.http.allowRestrictedHeaders=true" in JVM argument 24 | * * To use HTTPS Proxy Authorization, due to the HTTPS tunnel BASIC authentication has been disabled by default since JDK8u111, you may need to add "-Djdk.http.auth.tunneling.disabledSchemes=" in JVM argument 25 | * * To add JVM arguments In TOMCAT, modify catalina.bat/catalina.sh 26 | */ 27 | @SuppressWarnings({"rawtypes", "unchecked", "serial", "UnusedReturnValue", "WeakerAccess", "unused", "JavaDoc"}) 28 | public final class CUrl { 29 | private static final String VERSION = "1.2.2"; 30 | private static final String DEFAULT_USER_AGENT = "Java-CURL version " + VERSION + " by Rocks Wang(https://github.com/rockswang)"; 31 | private static final Pattern ptnOptionName = Pattern.compile("-{1,2}[a-zA-Z][a-zA-Z0-9\\-.]*"); 32 | private static final CookieStore cookieStore = new CookieStore(); 33 | private static HostnameVerifier insecureVerifier = null; 34 | private static SSLSocketFactory insecureFactory = null; 35 | private static boolean verbose = false; 36 | private static String trustStoreFile = "system"; // default to system trustmanager 37 | 38 | static { 39 | try { 40 | // Try to enable the setting to restricted headers like "Origin", this is expected to be executed before HttpURLConnection class-loading 41 | System.setProperty("sun.net.http.allowRestrictedHeaders", "true"); 42 | 43 | // Modify the system-wide Cookie manager to ThreadLocal-based instance 44 | CookieManager.setDefault(new CookieManager(cookieStore, CookiePolicy.ACCEPT_ALL)); 45 | 46 | // For insecure HTTPS 47 | insecureVerifier = new HostnameVerifier() { 48 | public boolean verify(String hostname, SSLSession session) { return true; } 49 | }; 50 | insecureFactory = getSocketFactory(null, null, true); 51 | } catch (Exception ignored) {} 52 | } 53 | 54 | private static final Map optMap = Util.mapPut(new LinkedHashMap(), 55 | "-E", 32, 56 | "--cert", 32, // Client certificate file and password 57 | "--cacert", 34, // Truststore to verify server certs, default to system truststore 58 | "--compressed", 1, // Request compressed response (using deflate or gzip) 59 | "--connect-timeout", 2, // SECONDS Maximum time allowed for connection 60 | "-b", 3, 61 | "--cookie", 3, // STRING/FILE Read cookies from STRING/FILE (H) 62 | "-c", 4, 63 | "--cookie-jar", 4, // FILE Write cookies to FILE after operation (H) 64 | "-d", 5, 65 | "--data", 5, // DATA HTTP POST data (H) 66 | "--data-ascii", 5, // DATA HTTP POST ASCII data (H) 67 | "--data-raw", 51, // DATA HTTP POST raw data (H) 68 | "--data-binary", 52, // DATA HTTP POST binary data (H) 69 | "--data-urlencode", 53, // DATA HTTP POST data url encoded (H) 70 | "-D", 6, 71 | "--dump-header", 6, // FILE Write the headers to FILE 72 | "-F", 7, 73 | "--form", 7, // CONTENT Specify HTTP multipart POST data (H) 74 | "--form-string", 71, // STRING Specify HTTP multipart POST data (H) 75 | "-G", 8, 76 | "--get", 8, // Send the -d data with a HTTP GET (H) 77 | "-H", 10, 78 | "--header", 10, // LINE Pass custom header LINE to server (H) 79 | "-I", 11, 80 | "--head", 11, // Show document info only 81 | // "--ignore-content-length", 12, // Ignore the HTTP Content-Length header 82 | "-k", 31, 83 | "--insecure", 31, // Allow insecure server connections when using SSL 84 | "-L", 13, 85 | "--location", 13, // Follow redirects (H) 86 | "-m", 14, 87 | "--max-time", 14, // SECONDS Maximum time allowed for the transfer 88 | // "--no-keepalive", 15, // Disable keepalive use on the connection 89 | "-o", 16, 90 | "--output", 16, // FILE Write to FILE instead of stdout 91 | "-x", 17, 92 | "--proxy", 17, // [PROTOCOL://]HOST[:PORT] Use proxy on given port 93 | "-U", 18, 94 | "--proxy-user", 18, // USER[:PASSWORD] Proxy user and password 95 | "-e", 19, 96 | "--referer", 19, // Referer URL (H) 97 | "--retry", 20, // NUM Retry request NUM times if transient problems occur 98 | "--retry-delay", 21, // SECONDS Wait SECONDS between retries 99 | "--retry-max-time", 22, // SECONDS Retry only within this period 100 | "-s", 23, 101 | "--silent", 23, // Silent mode (don't output anything) 102 | "--stderr", 24, // FILE Where to redirect stderr (use "-" for stdout) 103 | "-u", 28, 104 | "--user", 28, // USER[:PASSWORD] Server user and password 105 | "--url", 25, // URL URL to work with 106 | "-A", 26, 107 | "--user-agent", 26, // STRING Send User-Agent STRING to server (H) 108 | "-X", 27, 109 | "--request", 27, // COMMAND Specify request command to use 110 | "--x-max-download", 29, // BYTES Maximum bytes allowed for the download 111 | "--x-tags", 30, // DATA extra key-value pairs, storage only 112 | "--verbose", 33, // Verbose 113 | "-v", 33, // Verbose 114 | "", 0 // placeholder 115 | ); 116 | 117 | private static final String BOUNDARY = "------------aia113jBkadk7289"; 118 | private static final byte[] NEWLINE = "\r\n".getBytes(); 119 | 120 | private final List options = new ArrayList(); 121 | private final Map iomap = new HashMap(); 122 | private final Map tags = new LinkedHashMap(); 123 | private final Map headers = new LinkedHashMap(); 124 | private final List> responseHeaders = new ArrayList>(4); 125 | private final List locations = new ArrayList(4); 126 | private long startTime; 127 | private long execTime; 128 | private int httpCode; 129 | private byte[] rawStdout; 130 | 131 | public CUrl() {} 132 | 133 | public CUrl(String url) { 134 | this.url(url); 135 | } 136 | 137 | /** 138 | * Specify 0~N options, please refer to https://curl.haxx.se/docs/manpage.html 139 | * Note: the option name and corresponding value must be divided into two arguments, rather than one single string seperated by space 140 | * @param options e.g. opt("-H", "X-Requested-With: XMLHttpRequest") 141 | */ 142 | public final CUrl opt(String... options) { 143 | for (String o: options) { 144 | if (o.startsWith("'") && o.endsWith("'")) o = o.substring(1, o.length() - 1); 145 | this.options.add(o); 146 | } 147 | return this; 148 | } 149 | 150 | public final CUrl url(String url) { 151 | return opt("--url", url); 152 | } 153 | 154 | /** 155 | * Follow redirection automatically, false be default. 156 | * Only apply to HTTP CODE 30x 157 | */ 158 | public final CUrl location() { 159 | return opt("-L"); 160 | } 161 | 162 | /** 163 | * Specify the proxy server 164 | */ 165 | public final CUrl proxy(String host, int port) { 166 | return opt("-x", host + ":" + port); 167 | } 168 | 169 | /** 170 | * Allow insecure server connections when using HTTPS 171 | */ 172 | public final CUrl insecure() { 173 | return opt("-k"); 174 | } 175 | 176 | /** 177 | * Verbose output 178 | */ 179 | public final CUrl verbose() { 180 | return opt("-v"); 181 | } 182 | 183 | /** 184 | * Specify retry related options, default values are 0 185 | * @param retry Retry times 186 | * @param retryDelay The interval between two retries (in second) 187 | * @param retryMaxTime The max retry time in second, 0 means infinite 188 | */ 189 | public final CUrl retry(int retry, float retryDelay, float retryMaxTime) { 190 | return opt("--retry", Integer.toString(retry), 191 | "--retry-delay", Float.toString(retryDelay), 192 | "--retry-max-time", Float.toString(retryMaxTime)); 193 | } 194 | 195 | /** 196 | * Specify timeout, default values are 0 197 | * @param connectTimeoutSeconds Connection timeout in second 198 | * @param readTimeoutSeconds Read timeout in second 199 | */ 200 | public final CUrl timeout(float connectTimeoutSeconds, float readTimeoutSeconds) { 201 | return opt("--connect-timeout", Float.toString(connectTimeoutSeconds), 202 | "--max-time", Float.toString(readTimeoutSeconds)); 203 | } 204 | 205 | /** 206 | * Add a custom request header 207 | * @param headerLine Syntax: 208 | * "Host: baidu.com": add/set a request header-value pair 209 | * "Accept:": delete a previously added request header 210 | * "X-Custom-Header;": add/set a request header with empty value 211 | */ 212 | public final CUrl header(String headerLine) { 213 | return opt("-H", headerLine); 214 | } 215 | 216 | public final CUrl headers(Map headers) { 217 | for (Map.Entry kv: headers.entrySet()) { 218 | Object k = kv.getKey(), v = kv.getValue(); 219 | opt("-H", v == null ? k + ":" : v.toString().length() == 0 ? k + ";" : k + ": " + v); 220 | } 221 | return this; 222 | } 223 | 224 | /** 225 | * Add post data. The data among multiple calls will be joined with '&' 226 | * @param data if data start with '@', then the following part will be treated as file path 227 | */ 228 | public final CUrl data(String data) { 229 | return data(data, false); 230 | } 231 | 232 | /** 233 | * Add post data. The data among multiple calls will be joined with '&' 234 | * @param data 如果data以'@'开头且raw=false，则后面部分作为文件名，数据由该文件读入 235 | * @param raw 如为真则不对'@'做特殊处理 236 | */ 237 | public final CUrl data(String data, boolean raw) { 238 | return opt(raw ? "--data-raw" : "-d", data); 239 | } 240 | 241 | /** 242 | * 从input中读取数据作为post数据 243 | * Read data from input and use as post data 244 | * @param input 245 | * @param binary 如为真则读取数据中的回车换行符会保留，否则会被删除 246 | */ 247 | public final CUrl data(IO input, boolean binary) { 248 | String key; 249 | iomap.put(key = "IO#" + iomap.size(), input); 250 | return opt(binary ? "--data-binary" : "-d", "@" + key); 251 | } 252 | 253 | /** 254 | * 添加urlencode的post数据 255 | * Add URL-encoded post data 256 | * @param data, 语法如下/syntax: 257 | * "content": 如content中不包含'=', '@'，则直接把整个content作为数据整体进行urlencode 258 | * "=content": '='后面的content整体进行urlencode，不处理特殊字符，第一个'='不包含在数据内容中 259 | * "name1=value1[&name2=value2...]": 按照'&'拆分后，对每个值进行urlencode，注意键不进行处理 260 | * "@filename": '@'后面的部分作为文件名，从文件中读取内容并进行urlencode，回车换行保留 261 | * "name@filename": 读取'@'后面的文件内容作为值进行urlencode，并以name为键 262 | * @param charset urlencode使用的字符集，如为null则默认使用"UTF-8" 263 | */ 264 | public final CUrl data(String data, String charset) { 265 | return opt("--data-urlencode" + (charset != null ? "-" + charset : ""), data); 266 | } 267 | 268 | /** 269 | * 发起post文件上传，添加一个表单项 270 | * Issue a form based upload and add a form item 271 | * @param name 表单项名 272 | * @param content 如首字母为'@'或'<'则从指定的文件读取数据进行上传。 273 | * '@'和'<'的区别在于，'@'的文件内容作为文件附件上传，'<'的文件内容作为普通表单项 274 | */ 275 | public final CUrl form(String name, String content) { 276 | return opt("-F", name + "=" + content); 277 | } 278 | 279 | /** 280 | * 发起post文件上传，添加一个文件上传的表单项 281 | * Issue a form based upload and add a file item 282 | * @param name 表单项名 283 | * @param input 上传的数据IO 284 | */ 285 | public final CUrl form(String name, IO input) { 286 | String key; 287 | iomap.put(key = "IO#" + iomap.size(), input); 288 | return opt("-F", name + "=@" + key); 289 | } 290 | 291 | /** 292 | * 发起post文件上传，添加1~N个非文件表单项，注意此方法不对'@'进行特殊处理 293 | * @param formString 语法为"name1=value1[&name2=value2...]" 294 | */ 295 | public final CUrl form(String formString) { 296 | return opt("--form-string", formString); 297 | } 298 | 299 | /** 300 | * 输出Cookie到给定的文件 301 | * Output Cookie to given file path 302 | * @param output 文件路径，使用'-'表示输出到标准输出。默认不输出 303 | */ 304 | public final CUrl cookieJar(String output) { 305 | return opt("-c", output); 306 | } 307 | 308 | /** 309 | * 输出Cookie到给定的数据IO 310 | * Output Cookie to given IO object 311 | * @param output 数据IO，注意cookieJar的输出会清除output中的原有内容 312 | */ 313 | public final CUrl cookieJar(IO output) { 314 | String key; 315 | iomap.put(key = "IO#" + iomap.size(), output); 316 | return opt("-c", key); 317 | } 318 | 319 | /** 320 | * 添加请求Cookie 321 | * Add custom Cookies in request 322 | * @param input 格式为"NAME1=VALUE1; NAME2=VALUE2"的Cookie键值对。 323 | * 如字串中不包含'='则作为输入文件名； 324 | * 如传入空字符串则仅清空当前线程的Cookie 325 | */ 326 | public final CUrl cookie(String input) { 327 | return opt("-b", input); 328 | } 329 | 330 | /** 331 | * 读取数据IO并添加请求Cookie。 332 | * 注意CUrl会自动为同一线程内的多次请求维持Cookie 333 | * @param input 334 | * @return 335 | */ 336 | public final CUrl cookie(IO input) { 337 | String key; 338 | iomap.put(key = "IO#" + iomap.size(), input); 339 | return opt("-b", key); 340 | } 341 | 342 | /** 343 | * 倾印原始响应头到给定的文件 344 | * Dump raw response headers to specified file path 345 | * @param output 输出文件的路径，使用'-'表示输出到标准输出。默认不输出。 346 | */ 347 | public final CUrl dumpHeader(String output) { 348 | return opt("-D", output); 349 | } 350 | 351 | /** 352 | * 倾印原始响应头到给定的数据IO 353 | * @param output 354 | */ 355 | public final CUrl dumpHeader(IO output) { 356 | String key; 357 | iomap.put(key = "IO#" + iomap.size(), output); 358 | return opt("-D", key); 359 | } 360 | 361 | public final CUrl cert(IO certificate, String password) { 362 | String key; 363 | iomap.put(key = "IO#" + iomap.size(), certificate); 364 | return opt("-E", key + ":" + password); 365 | } 366 | 367 | /** 368 | * 重定向标准错误输出到给定的文件 369 | * Redirect stderr to specified file path, use '-' for stdout 370 | * @param output 输出文件路径。使用'-'表示输出到标准输出。默认输出到标准输出。 371 | */ 372 | public final CUrl stderr(String output) { 373 | return opt("--stderr", output); // output can be an OutputStream/File/path_to_file 374 | } 375 | 376 | /** 377 | * 重定向标准错误输出到给定的数据IO 378 | * @param output 379 | */ 380 | public final CUrl stderr(IO output) { 381 | String key; 382 | iomap.put(key = "IO#" + iomap.size(), output); 383 | return opt("--stderr", key); 384 | } 385 | 386 | /** 387 | * 输出应答数据到给定文件。注意标准输出默认即为exec方法的返回值。 388 | * Output response data to specified fila path 389 | * @param output 输出文件路径。使用'-'表示输出到标准输出。默认输出到标准输出。 390 | */ 391 | public final CUrl output(String output) { 392 | return opt("-o", output); // output can be an OutputStream/File/path_to_file 393 | } 394 | 395 | /** 396 | * 输出应答数据到给定数据IO 397 | * @param output 398 | */ 399 | public final CUrl output(IO output) { 400 | String key; 401 | iomap.put(key = "IO#" + iomap.size(), output); 402 | return opt("-o", key); 403 | } 404 | 405 | /** 406 | * 添加一个数据IO，可作为数据输入或数据输出，在--data等参数值中引用 407 | * @param key 408 | * @param io 409 | * @return 410 | */ 411 | public final CUrl io(String key, IO io) { 412 | iomap.put(key, io); 413 | return this; 414 | } 415 | 416 | public static java.net.CookieStore getCookieStore() { 417 | return cookieStore; 418 | } 419 | 420 | /** 421 | * Save all cookies binding with current thread to the specified IO object. 422 | * The output format is compatible with CURL tool. 423 | * @param output 424 | */ 425 | public static void saveCookies(IO output) { 426 | if (output instanceof CookieIO) { 427 | CookieIO cs = (CookieIO) output; 428 | synchronized (cs) { for (HttpCookie c: cookieStore.getCookies()) cs.add(null, c); } 429 | } else { 430 | String s = dumpCookies(cookieStore.getCookies()); 431 | writeOutput(output, Util.s2b(s, null), false); 432 | } 433 | } 434 | 435 | public static String dumpCookie(HttpCookie cookie) { 436 | StringBuilder sb = new StringBuilder(); 437 | long expire = cookie.getMaxAge() <= 0 || cookie.getMaxAge() >= Integer.MAX_VALUE ? 438 | Integer.MAX_VALUE : cookie.getMaxAge() + System.currentTimeMillis() / 1000L; 439 | sb.append(cookie.getDomain()).append('\t') 440 | .append("FALSE").append('\t') 441 | .append(cookie.getPath()).append('\t') 442 | .append(cookie.getSecure() ? "TRUE" : "FALSE").append('\t') 443 | .append(expire).append('\t') 444 | .append(cookie.getName()).append('\t') 445 | .append(cookie.getValue()).append('\n'); 446 | return sb.toString(); 447 | } 448 | 449 | public static String dumpCookies(List cookies) { 450 | StringBuilder sb = new StringBuilder(); 451 | for (HttpCookie cookie: cookies) sb.append(dumpCookie(cookie)); 452 | return sb.toString(); 453 | } 454 | 455 | /** 456 | * Load cookies from the specified IO object to the cookie-store binding with current thread 457 | * @param input 458 | */ 459 | public static void loadCookies(IO input) { 460 | if (input instanceof CookieIO) { 461 | CookieIO cs = (CookieIO) input; 462 | synchronized (cs) { for (HttpCookie c: cs.getCookies()) cookieStore.add(null, c); } 463 | } else { 464 | List cookies = parseCookies(Util.b2s(readInput(input), null, null)); 465 | for (HttpCookie c : cookies) cookieStore.add(null, c); 466 | } 467 | } 468 | 469 | public static List parseCookies(String input) { 470 | BufferedReader br = new BufferedReader(new StringReader(input)); 471 | ArrayList result = new ArrayList(); 472 | try { 473 | for (String line = br.readLine(), l[]; line != null; line = br.readLine()) { 474 | if (line.trim().length() == 0 || line.startsWith("# ") || (l = line.split("\t")).length < 7) continue; 475 | HttpCookie cookie = new HttpCookie(l[5], l[6]); 476 | cookie.setDomain(l[0]); 477 | cookie.setPath(l[2]); 478 | cookie.setSecure("TRUE".equals(l[3])); 479 | long expire = Long.parseLong(l[4]); 480 | cookie.setMaxAge(expire >= Integer.MAX_VALUE ? Integer.MAX_VALUE : expire * 1000L - System.currentTimeMillis()); 481 | if (!cookie.hasExpired()) cookieStore.add(null, cookie); 482 | } 483 | } catch (Exception ignored) { } // should not happen 484 | return result; 485 | } 486 | 487 | /** 488 | * Get all options as CURL command-line 489 | */ 490 | public final String toString() { 491 | StringBuilder sb = new StringBuilder("curl"); 492 | for (String s: options) { 493 | sb.append(' ').append(ptnOptionName.matcher(s).matches() ? s : '"' + s + '"'); 494 | } 495 | if (iomap.size() > 0) sb.append("\r\n> IOMap: ").append(iomap); 496 | return sb.toString(); 497 | } 498 | 499 | /** 500 | * Get all options, filled after exec. 501 | * You can change one or more options and re-exec the same CUrl instance, for example, switch proxy server. 502 | * @return 503 | */ 504 | public final List getOptions() { 505 | return options; 506 | } 507 | 508 | public final String getOption(String name) { 509 | if (!ptnOptionName.matcher(name).matches()) throw new IllegalArgumentException("Invalid option name: " + name); 510 | int idx = options.indexOf(name); 511 | if (idx < 0) return null; 512 | String next = idx + 1 < options.size() ? options.get(idx + 1) : null; 513 | if (next != null && ptnOptionName.matcher(next).matches()) next = null; 514 | return next != null ? next : "TRUE"; 515 | } 516 | 517 | public final Map getTags() { 518 | return tags; 519 | } 520 | 521 | /** 522 | * Get request headers, filled after exec. 523 | * @return 524 | */ 525 | public final Map getHeaders() { 526 | return headers; 527 | } 528 | 529 | /** 530 | * Get headers of all responses including redirection(s) in one request. 531 | * In case --location is not specified (default), it's always exactly one element. 532 | * @return 533 | */ 534 | public final List> getResponseHeaders() { 535 | return responseHeaders; 536 | } 537 | 538 | /** 539 | * Get total time-consuming including retrying in millisecond. 540 | * @return 541 | */ 542 | public final long getExecTime() { 543 | return execTime; 544 | } 545 | 546 | /** 547 | * Get HTTP status code of last response, i.e. 200, 302 etc. 548 | * @return 549 | */ 550 | public final int getHttpCode() { 551 | return httpCode; 552 | } 553 | 554 | public final T getStdout(Resolver resolver, T fallback) { 555 | try { return resolver.resolve(httpCode, rawStdout); } catch (Throwable ignored) {} 556 | return fallback; 557 | } 558 | 559 | /** 560 | * Get all destination URLs including redirection(s) in one request. 561 | * In case --location is not specified (default), it's always exactly one element. 562 | * @return 563 | */ 564 | public final List getLocations() { 565 | return locations; 566 | } 567 | 568 | /** 569 | * 解析参数，执行请求，并将标准输出以给定的encoding解码为字符串 570 | * Parse options and execute the request。Decode the raw response to String with given character-encoding 571 | * @param encoding，如传入null则默认使用"UTF-8" 572 | * @return 标准输出数据，以encoding编码为字符串 573 | */ 574 | public final String exec(String encoding) { 575 | return exec(encoding != null ? new ToStringResolver(encoding) : UTF8, null); 576 | } 577 | 578 | /** 579 | * 解析参数，执行请求，返回原始字节数组 580 | * Parse options and execute the request, return raw response. 581 | * @return 标准输出数据 582 | */ 583 | public final byte[] exec() { 584 | return exec(RAW, null); 585 | } 586 | 587 | /** 588 | * 解析参数并执行请求 589 | * 默认仅包含应答数据。指定"--silent"参数则不输出。 590 | * @param resolver 输出解析器 591 | * @param fallback 默认返回值 592 | * @return 将标准输出中的数据使用解析器转换为对象。如失败，则返回fallback 593 | */ 594 | public final T exec(Resolver resolver, T fallback) { 595 | startTime = System.currentTimeMillis(); 596 | tags.clear(); 597 | headers.clear(); 598 | responseHeaders.clear(); 599 | locations.clear(); 600 | execTime = 0; 601 | httpCode = -1; 602 | rawStdout = null; 603 | Proxy proxy = Proxy.NO_PROXY; 604 | String url = null, redirect = null, method = null, cookie = null, charset = "UTF-8", cert = null; 605 | final MemIO stdout = new MemIO(); 606 | IO stderr = stdout, output = stdout, cookieJar = null, dumpHeader = null; 607 | StringBuilder dataSb = new StringBuilder(); 608 | Map> form = new LinkedHashMap>(); 609 | float connectTimeout = 0, maxTime = 0, retryDelay = 0, retryMaxTime = 0; 610 | int retry = 0, maxDownload = 0; 611 | boolean location = false, silent = false, mergeData = false, insecure = false; 612 | // boolean ignoreContentLength = false, noKeepAlive = false; 613 | Util.mapPut(headers, "Accept", "*/*", "User-Agent", DEFAULT_USER_AGENT); 614 | iomap.put("-", stdout); 615 | Throwable lastEx = null; 616 | for (int i = 0, n = options.size(); i < n; i++) { 617 | String opt = options.get(i); 618 | if (opt.startsWith("http://") || opt.startsWith("https://")) { 619 | url = opt; 620 | continue; 621 | } 622 | if (opt.startsWith("--data-urlencode-")) { 623 | charset = opt.substring(17); 624 | opt = "--data-urlencode"; 625 | } 626 | switch (Util.mapGet(optMap, opt, -1)) { 627 | case 32: // --cert Client certificate file and password 628 | cert = options.get(++i); 629 | break; 630 | case 1: // --compressed Request compressed response (using deflate or gzip) 631 | headers.put("Accept-Encoding", "gzip, deflate"); 632 | break; 633 | case 2: // --connect-timeout SECONDS Maximum time allowed for connection 634 | connectTimeout = Float.parseFloat(options.get(++i)); 635 | break; 636 | case 3: // --cookie STRING/FILE Read cookies from STRING/FILE (H) 637 | cookie = options.get(++i); 638 | break; 639 | case 4: // --cookie-jar FILE Write cookies to FILE after operation (H) 640 | cookieJar = getIO(options.get(++i)); 641 | break; 642 | case 5: // --data DATA HTTP POST data (H) 643 | String data = options.get(++i); 644 | if (data.startsWith("@")) data = Util.b2s(readInput(getIO(data.substring(1))), null, null).replaceAll("[\r\n]+", ""); 645 | mergeData = dataSb.length() > 0; 646 | dataSb.append(mergeData ? "&" : "").append(data); 647 | break; 648 | case 51: // --data-raw DATA not handle "@" 649 | mergeData = dataSb.length() > 0; 650 | dataSb.append(mergeData ? "&" : "").append(options.get(++i)); 651 | break; 652 | case 52: // --data-binary DATA not stripping CR/LF 653 | data = options.get(++i); 654 | if (data.startsWith("@")) data = Util.b2s(readInput(getIO(data.substring(1))), null, null); 655 | mergeData = dataSb.length() > 0; 656 | dataSb.append(mergeData ? "&" : "").append(data); 657 | break; 658 | case 53: // --data-urlencode 659 | mergeData = dataSb.length() > 0; 660 | data = options.get(++i); 661 | int idx, atIdx; 662 | switch (idx = data.indexOf("=")) { 663 | case -1: // no '=' 664 | if ((atIdx = data.indexOf("@")) >= 0) { // [name]@filename 665 | String prefix = atIdx > 0 ? data.substring(0, atIdx) + "=" : ""; 666 | try { 667 | data = prefix + URLEncoder.encode(Util.b2s(readInput(getIO(data.substring(atIdx + 1))), null, ""), charset); 668 | } catch (Exception e) { 669 | lastEx = e; 670 | } 671 | break; 672 | } // else fall through 673 | case 0: // =content 674 | try { 675 | data = URLEncoder.encode(data.substring(idx + 1), charset); 676 | } catch (Exception e) { 677 | lastEx = e; 678 | } 679 | break; 680 | default: // name=content 681 | Map m = Util.split(data, "&", "=", new LinkedHashMap()); 682 | for (Map.Entry en: m.entrySet()) { 683 | try { en.setValue(URLEncoder.encode(en.getValue(), "UTF-8")); } catch (Exception ignored) { } 684 | } 685 | data = Util.join(m, "&", "="); 686 | } 687 | dataSb.append(mergeData ? "&" : "").append(data); 688 | break; 689 | case 6: // --dump-header FILE Write the headers to FILE 690 | dumpHeader = getIO(options.get(++i)); 691 | break; 692 | case 7: // --form CONTENT Specify HTTP multipart POST data (H) 693 | data = options.get(++i); 694 | idx = data.indexOf('='); 695 | form.put(data.substring(0, idx), new Util.Ref(1, data.substring(idx + 1))); 696 | break; 697 | case 71: // --form-string STRING Specify HTTP multipart POST data (H) 698 | for (String[] pair: Util.split(options.get(++i), "&", "=")) { 699 | form.put(pair[0], new Util.Ref(pair[1])); 700 | } 701 | break; 702 | case 8: // --get Send the -d data with a HTTP GET (H) 703 | method = "GET"; 704 | break; 705 | case 10: // --header LINE Pass custom header LINE to server (H) 706 | String[] hh = options.get(++i).split(":", 2); 707 | String name = hh[0].trim(); 708 | if (hh.length == 1 && name.endsWith(";")) { // "X-Custom-Header;" 709 | headers.put(name.substring(0, name.length() - 1), ""); 710 | } else if (hh.length == 1 || Util.empty(hh[1])) { // "Host:" 711 | headers.remove(name); 712 | } else { // "Host: baidu.com" 713 | headers.put(name, hh[1].trim()); 714 | } 715 | break; 716 | case 11: // --head Show document info only 717 | method = "HEAD"; 718 | break; 719 | // case 12: // --ignore-content-length Ignore the HTTP Content-Length header 720 | // ignoreContentLength = true; 721 | // break; 722 | case 13: // --location Follow redirects (H) 723 | location = true; 724 | break; 725 | case 14: // --max-time SECONDS Maximum time allowed for the transfer 726 | maxTime = Float.parseFloat(options.get(++i)); 727 | break; 728 | // case 15: // --no-keepalive Disable keepalive use on the connection 729 | // noKeepAlive = true; 730 | // break; 731 | case 16: // --output FILE Write to FILE instead of stdout 732 | output = getIO(options.get(++i)); 733 | break; 734 | case 17: // --proxy [PROTOCOL://]HOST[:PORT] Use proxy on given port 735 | String[] pp = options.get(++i).split(":"); 736 | InetSocketAddress addr = new InetSocketAddress(pp[0], pp.length > 1 ? Integer.parseInt(pp[1]) : 1080); 737 | proxy = new Proxy(Proxy.Type.HTTP, addr); 738 | break; 739 | case 18: // --proxy-user USER[:PASSWORD] Proxy user and password 740 | final String proxyAuth = options.get(++i); 741 | headers.put("Proxy-Authorization", "Basic " + Util.base64Encode(proxyAuth.getBytes())); 742 | Authenticator.setDefault(new Authenticator() { 743 | @Override 744 | protected PasswordAuthentication getPasswordAuthentication() { 745 | String[] up = proxyAuth.split(":"); 746 | return new PasswordAuthentication(up[0], (up.length > 1 ? up[1] : "").toCharArray()); 747 | } 748 | }); 749 | break; 750 | case 19: // --referer Referer URL (H) 751 | headers.put("Referer", options.get(++i)); 752 | break; 753 | case 20: // --retry NUM Retry request NUM times if transient problems occur 754 | retry = Integer.parseInt(options.get(++i)); 755 | break; 756 | case 21: // --retry-delay SECONDS Wait SECONDS between retries 757 | retryDelay = Float.parseFloat(options.get(++i)); 758 | break; 759 | case 22: // --retry-max-time SECONDS Retry only within this period 760 | retryMaxTime = Float.parseFloat(options.get(++i)); 761 | break; 762 | case 23: // --silent Silent mode (don't output anything) 763 | silent = true; 764 | break; 765 | case 24: // --stderr FILE Where to redirect stderr (use "-" for stdout) 766 | stderr = getIO(options.get(++i)); 767 | break; 768 | case 25: // --url URL URL to work with 769 | url = options.get(++i); 770 | break; 771 | case 26: // --user-agent STRING Send User-Agent STRING to server (H) 772 | headers.put("User-Agent", options.get(++i)); 773 | break; 774 | case 27: // --request COMMAND Specify request command to use 775 | method = options.get(++i); 776 | break; 777 | case 28: // -u, --user USER[:PASSWORD] Server user and password 778 | headers.put("Authorization", "Basic " + Util.base64Encode(options.get(++i).getBytes())); 779 | break; 780 | case 29: // --x-max-download BYTES Maximum bytes allowed for the download 781 | maxDownload = Integer.parseInt(options.get(++i)); 782 | break; 783 | case 30: // --x-tags DATA extra key-value pairs, storage only 784 | Util.split(options.get(++i), "&", "=", tags); 785 | break; 786 | case 31: // 787 | insecure = true; 788 | break; 789 | case 33: // 790 | verbose = true; 791 | break; 792 | case 34: // --cacert file_name to specify the truststore file, "system" for system truststore 793 | trustStoreFile = options.get(++i); 794 | break; 795 | 796 | default: lastEx = new IllegalArgumentException("option " + opt + ": is unknown"); 797 | } 798 | if (lastEx != null) 799 | return error(stdout, stderr, lastEx, silent, resolver, fallback); 800 | } 801 | if (url == null) { 802 | lastEx = new IllegalArgumentException("no URL specified!"); 803 | return error(stdout, stderr, lastEx, silent, resolver, fallback); 804 | } 805 | if (dataSb.length() > 0 && form.size() > 0 806 | || dataSb.length() > 0 && "HEAD".equals(method) 807 | || form.size() > 0 && "HEAD".equals(method)) { 808 | lastEx = new IllegalArgumentException("Warning: You can only select one HTTP request!"); 809 | return error(stdout, stderr, lastEx, silent, resolver, fallback); 810 | } 811 | String dataStr = ""; 812 | if (form.size() > 0) { 813 | if (method == null) method = "POST"; 814 | } else if (dataSb.length() > 0) { 815 | dataStr = !mergeData ? dataSb.toString() 816 | : Util.join(Util.split(dataSb.toString(), "&", "=", new LinkedHashMap()), "&", "="); 817 | if (method == null) method = "POST"; 818 | } 819 | if (method == null) method = "GET"; 820 | // if (!noKeepAlive) headers.put("Connection", "keep-alive"); 821 | if (cookie != null) { // --cookie '' will clear the CookieStore 822 | cookieStore.removeAll(); 823 | if (cookie.indexOf('=') > 0) { 824 | parseCookies(url, cookie); 825 | } else if (cookie.trim().length() > 0) { 826 | loadCookies(getIO(cookie)); 827 | } 828 | } 829 | 830 | boolean needRetry = false; 831 | if (dataStr.length() > 0 && "GET".equals(method)) url += (url.contains("?") ? "&" : "?") + dataStr; 832 | URL urlObj = null; 833 | int retryLeft = retry; 834 | do { 835 | try { 836 | if (redirect != null) { 837 | urlObj = new URL(urlObj, redirect); 838 | method = "GET"; 839 | } else { 840 | urlObj = new URL(url); 841 | } 842 | if (retryLeft == retry) { // add at first time 843 | if (locations.size() > 51) { 844 | redirect = null; 845 | throw new RuntimeException("Too many redirects."); 846 | } 847 | locations.add(urlObj); 848 | responseHeaders.add(new ArrayList()); 849 | } 850 | if (verbose) { 851 | Util.logStderr("Prepare open connection - URL.openConnection()"); 852 | } 853 | HttpURLConnection con = (HttpURLConnection) urlObj.openConnection(proxy); 854 | if (verbose) { 855 | Util.logStderr("Done prepare open connection"); 856 | } 857 | con.setRequestMethod(method); 858 | con.setUseCaches(false); 859 | con.setConnectTimeout((int) (connectTimeout * 1000f)); 860 | con.setReadTimeout((int) (maxTime * 1000f)); 861 | con.setInstanceFollowRedirects(false); 862 | if (con instanceof HttpsURLConnection) { 863 | if (insecure) { 864 | Util.logStderr("Skip TLS validation"); 865 | ((HttpsURLConnection) con).setHostnameVerifier(insecureVerifier); 866 | ((HttpsURLConnection) con).setSSLSocketFactory(insecureFactory); 867 | } else { 868 | String keyStoreFn = null; 869 | String keyStorePass = null; 870 | if ( cert != null) { 871 | Util.logStderr("Enable client cert"); 872 | int idx = cert.lastIndexOf(':'); 873 | keyStoreFn = cert.substring(0, idx); 874 | keyStorePass = cert.substring(idx + 1); 875 | } 876 | ((HttpsURLConnection) con).setSSLSocketFactory(getSocketFactory(getIO(keyStoreFn), keyStorePass, false)); 877 | } 878 | } 879 | if (verbose) { 880 | Util.logStderr("Prepare headers"); 881 | } 882 | for (Map.Entry h: headers.entrySet()) con.setRequestProperty(h.getKey(), h.getValue()); 883 | if (verbose) { 884 | Util.logStderr("Done preparing headers"); 885 | } 886 | if ("POST".equals(method) || "PUT".equals(method)) { 887 | con.setDoInput(true); 888 | con.setDoOutput(true); 889 | byte[] data; 890 | if (form.size() > 0) { // it's upload 891 | con.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + BOUNDARY); 892 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 893 | byte[] bb; 894 | for (Map.Entry> en: form.entrySet()) { 895 | String name = en.getKey(), filename = null, type = null; 896 | Util.Ref val = en.getValue(); 897 | if (val.getInt() == 1) { 898 | String[][] ll = Util.split(val.get(), ";", "="); 899 | String _1st = unquote(ll[0][0].trim()); 900 | for (int j = 1; j < ll.length; j++) { 901 | if (ll[j].length > 1 && "type".equals(ll[j][0].trim())) { 902 | type = unquote(ll[j][1].trim()); 903 | } else if (ll[j].length > 1 && "filename".equals(ll[j][0].trim())) { 904 | filename = unquote(ll[j][1].trim()); 905 | } 906 | } 907 | if (_1st.startsWith("@") || _1st.startsWith("<")) { // it's file 908 | IO in = getIO(_1st.substring(1)); 909 | File f = in instanceof FileIO ? ((FileIO) in).f : null; 910 | filename = _1st.startsWith("<") ? null : 911 | filename != null ? filename : f != null ? f.getAbsolutePath() : name; 912 | if (f != null && !(f.exists() && f.isFile() && f.canRead())) 913 | throw new IllegalArgumentException("couldn't open file \"" + filename + "\""); 914 | bb = readInput(in); 915 | } else { 916 | bb = Util.s2b(_1st, null); 917 | } 918 | } else { 919 | bb = Util.s2b(val.get(), null); 920 | } 921 | os.write(("--" + BOUNDARY + "\r\n").getBytes()); 922 | os.write(("Content-Disposition: form-data; name=\"" + name + "\"").getBytes()); 923 | if (filename != null) os.write(("; filename=\"" + filename + "\"").getBytes()); 924 | if (type != null) os.write(("\r\nContent-Type: " + type).getBytes()); 925 | os.write(NEWLINE); 926 | os.write(NEWLINE); 927 | os.write(bb); 928 | os.write(NEWLINE); 929 | } 930 | os.write(("--" + BOUNDARY + "--\r\n").getBytes()); 931 | data = os.toByteArray(); 932 | } else { 933 | data = Util.s2b(dataStr, null); // UTF-8 934 | // if (!ignoreContentLength) { 935 | con.setRequestProperty("Content-Length", Integer.toString(data.length)); 936 | if (!headers.containsKey("Content-Type")) { 937 | con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); 938 | } 939 | } 940 | try { 941 | if (verbose) { 942 | Util.logStderr("Start getOutputStream"); 943 | } 944 | OutputStream os = con.getOutputStream(); 945 | os.write(data); 946 | os.flush(); 947 | if (verbose) { 948 | Util.logStderr("Done sending data"); 949 | } 950 | } catch (Exception ex) { // connect timeout 951 | throw new Recoverable(ex, -1); 952 | } 953 | } 954 | redirect = null; 955 | if (verbose) { 956 | Util.logStderr("Get HTTP Response Code - HttpURLConnection.getResponseCode()"); 957 | } 958 | httpCode = con.getResponseCode(); 959 | if (httpCode >= 300 && httpCode < 400) redirect = con.getHeaderField("Location"); 960 | if (verbose) { 961 | Util.logStderr("HTTP Response Code: %d", httpCode); 962 | } 963 | if (redirect != null) retryLeft = retry; 964 | InputStream is; 965 | try { 966 | if (verbose) { 967 | Util.logStderr("Getting input stream"); 968 | } 969 | is = con.getInputStream(); 970 | } catch (Exception e) { 971 | if (httpCode == 407 && proxy != Proxy.NO_PROXY && "https".equals(urlObj.getProtocol()) 972 | && headers.containsKey("Proxy-Authorization")) { 973 | throw new RuntimeException(e.getMessage() + "\nTry using VM argument \"-Djdk.http.auth.tunneling.disabledSchemes=\"", e); 974 | } 975 | if (redirect == null) lastEx = e; 976 | is = con.getErrorStream(); 977 | } 978 | if (is == null && lastEx != null) throw lastEx; 979 | if (verbose) { 980 | Util.logStderr("Start reading output"); 981 | } 982 | byte bb[] = Util.readStream(is, maxDownload, true), b0, b1; 983 | if (verbose) { 984 | Util.logStderr("Done reading output"); 985 | } 986 | if (maxDownload <= 0 && bb != null && bb.length > 2) { 987 | if ((b0 = bb[0]) == 0x1F && bb[1] == (byte) 0x8B) is = new GZIPInputStream(new ByteArrayInputStream(bb)); // gzip 988 | if (b0 == 0x78 && ((b1 = bb[1]) == 0x01 || b1 == 0x5E || b1 == (byte) 0x9C || b1 == (byte) 0xDA)) is = new InflaterInputStream(new ByteArrayInputStream(bb)); // deflate/zlib 989 | if (is instanceof InflaterInputStream) bb = Util.readStream(is, false); 990 | } 991 | int idx = locations.size() - 1; 992 | fillResponseHeaders(con, responseHeaders.get(idx)); 993 | if (dumpHeader != null) dumpHeader(responseHeaders.get(idx), dumpHeader); 994 | if (bb != null && bb.length > 0) writeOutput(output, bb, output == dumpHeader); 995 | if (lastEx != null) throw lastEx; 996 | if (redirect == null || !location) { 997 | rawStdout = stdout.toByteArray(); 998 | execTime = System.currentTimeMillis() - startTime; 999 | if (cookieJar != null) saveCookies(cookieJar); 1000 | return silent ? fallback : getStdout(resolver, fallback); 1001 | } 1002 | } catch (Throwable e) { 1003 | needRetry = isRecoverable(e.getClass()); 1004 | lastEx = e instanceof Recoverable ? e.getCause() : e; 1005 | if (needRetry && retryLeft > 0 && retryDelay > 0) 1006 | try { Thread.sleep((long) (retryDelay * 1000d)); } catch (Exception ignored) {} 1007 | } 1008 | } while (location && redirect != null || needRetry && --retryLeft >= 0 1009 | && (retryMaxTime <= 0 || System.currentTimeMillis() - startTime < (long) (retryMaxTime * 1000d))); 1010 | return error(stdout, stderr, lastEx, silent, resolver, fallback); 1011 | } 1012 | 1013 | /** 根据key获取对应IO，如果iomap中没有，则key作为文件路径创建一个FileIO */ 1014 | private IO getIO(String key) { 1015 | IO io; 1016 | if (key == null || key.isEmpty()) { 1017 | return null; 1018 | } 1019 | return (io = iomap.get(key)) == null ? new FileIO(key) : io; 1020 | } 1021 | 1022 | private T error(IO stdout, IO stderr, Throwable ex, boolean silent, Resolver rr, T fallback) { 1023 | writeOutput(stderr, Util.dumpStackTrace(ex, false).getBytes(), true); 1024 | httpCode = ex instanceof Recoverable ? ((Recoverable) ex).httpCode : -1; 1025 | rawStdout = ((MemIO) stdout).toByteArray(); 1026 | execTime = System.currentTimeMillis() - startTime; 1027 | return silent ? fallback : getStdout(rr, fallback); 1028 | } 1029 | 1030 | private static void parseCookies(String url, String input) { 1031 | String host = null; 1032 | try { host = new URI(url).getHost(); } catch (Exception ignored) { } 1033 | for (String[] pair: Util.split(input, ";", "=")) { 1034 | HttpCookie cookie = new HttpCookie(pair[0], Util.urlDecode(pair[1], "UTF-8")); 1035 | cookie.setDomain(host); 1036 | cookie.setPath("/"); 1037 | cookie.setSecure(false); 1038 | cookieStore.add(null, cookie); 1039 | } 1040 | } 1041 | 1042 | private static String unquote(String s) { 1043 | return s.startsWith("'") && s.endsWith("'") || s.startsWith("\"") && s.endsWith("\"") ? 1044 | s.substring(1, s.length() - 1) : s; 1045 | } 1046 | 1047 | private static void fillResponseHeaders(HttpURLConnection con, List headers) { 1048 | headers.clear(); 1049 | Object responses = Util.getField(con, null, "responses", null, true); // sun.net.www.MessageHeader 1050 | if (responses == null) { // con is sun.net.www.protocol.https.HttpsURLConnectionImpl 1051 | Object delegate = Util.getField(con, null, "delegate", null, true); 1052 | if (delegate != null) responses = Util.getField(delegate, null, "responses", null, true); 1053 | } 1054 | String[] keys, values; 1055 | Integer nkeys; 1056 | if (responses != null && (nkeys = (Integer) Util.getField(responses, null, "nkeys", null, true)) != null 1057 | && (keys = (String[]) Util.getField(responses, null, "keys", null, true)) != null 1058 | && (values = (String[]) Util.getField(responses, null, "values", null, true)) != null) { 1059 | for (int i = 0; i < nkeys; i++) headers.add(new String[] { keys[i], values[i] }); 1060 | } else { 1061 | try { headers.add(new String[] { null, con.getResponseMessage() }); } catch (Exception ignored) {} 1062 | for (int i = 0; ; i++) { 1063 | String k = con.getHeaderFieldKey(i), v = con.getHeaderField(i); 1064 | if (k == null && v == null) break; 1065 | headers.add(new String[] { k, v }); 1066 | } 1067 | } 1068 | } 1069 | 1070 | private static void dumpHeader(List headers, IO dumpHeader) throws Exception { 1071 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 1072 | for (String[] kv: headers) 1073 | bos.write(((kv[0] != null ? kv[0] + ": " : "") + (kv[1] != null ? kv[1] : "") + "\r\n").getBytes()); 1074 | bos.write(NEWLINE); 1075 | writeOutput(dumpHeader, bos.toByteArray(), false); 1076 | } 1077 | 1078 | /** 读取IO中的数据，如不适用或无数据则返回空数组 */ 1079 | private static byte[] readInput(IO in) { 1080 | InputStream is = in.getInputStream(); 1081 | byte[] bb; 1082 | if (is == null || (bb = Util.readStream(is, false)) == null) bb = new byte[0]; 1083 | in.close(); 1084 | return bb; 1085 | } 1086 | 1087 | /** 把数据输出到IO，如不适用则直接返回。如append为true则向数据IO添加，否则覆盖。*/ 1088 | private static void writeOutput(IO out, byte[] bb, boolean append) { 1089 | out.setAppend(append); 1090 | OutputStream os = out.getOutputStream(); 1091 | if (os == null) return; 1092 | try { 1093 | os.write(bb); 1094 | os.flush(); 1095 | } catch (Exception e) { 1096 | Util.logStderr("CUrl.writeOutput: out=%s,bb=%s,append=%s,ex=%s", out, bb, append, Util.dumpStackTrace(e, true)); 1097 | } 1098 | out.close(); 1099 | } 1100 | 1101 | private static final HashSet RECOVERABLES = Util.listAdd( 1102 | new HashSet(), 1103 | (Class) Recoverable.class, 1104 | ConnectException.class, 1105 | HttpRetryException.class, 1106 | SocketException.class, 1107 | SocketTimeoutException.class, 1108 | NoRouteToHostException.class); 1109 | 1110 | private static boolean isRecoverable(Class errCls) { 1111 | if (RECOVERABLES.contains(errCls)) return true; 1112 | for (Class re: RECOVERABLES) if (re.isAssignableFrom(errCls)) return true; 1113 | return false; 1114 | } 1115 | 1116 | private static X509TrustManager getTrustManager() throws KeyStoreException { 1117 | if ( trustStoreFile.equals("system") ) { 1118 | Util.logStderr("Load default trust manager"); // default behavior 1119 | return null; 1120 | } 1121 | try { 1122 | KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); 1123 | Util.logStderr("Build trust manager from file: " + trustStoreFile); 1124 | int idx = trustStoreFile.lastIndexOf(':'); 1125 | if (idx < 0) { 1126 | FileInputStream inputStream = new FileInputStream(String.valueOf(trustStoreFile)); 1127 | trustStore.load(inputStream, null); 1128 | } else { 1129 | FileInputStream inputStream = new FileInputStream(String.valueOf(trustStoreFile.substring(0, idx))); 1130 | trustStore.load(inputStream, trustStoreFile.substring(idx + 1).toCharArray()); 1131 | } 1132 | TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); 1133 | trustManagerFactory.init(trustStore); 1134 | 1135 | TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); 1136 | if (trustManagers.length == 0) { 1137 | throw new KeyStoreException("No TrustManagers found"); 1138 | } 1139 | return Arrays.stream(trustManagers) 1140 | .filter(tm -> tm instanceof X509TrustManager ) 1141 | .map(tm -> (X509TrustManager) tm) 1142 | .findFirst() 1143 | .orElseThrow(() -> new KeyStoreException("No X509TrustManager found")); 1144 | } catch (IOException e) { 1145 | throw new KeyStoreException("TrustStore file not found or failed to load: " + e.getMessage()); 1146 | } catch (NoSuchAlgorithmException | CertificateException e) { 1147 | throw new KeyStoreException("Failed to parse truststore: " + e.getMessage()); 1148 | } 1149 | } 1150 | 1151 | private static SSLSocketFactory getSocketFactory(IO cert, String password, Boolean insecure) throws Exception { 1152 | TrustManager[] t_managers=null; 1153 | KeyManager[] k_managers=null; 1154 | t_managers = new TrustManager[] { new X509TrustManager() { 1155 | public X509Certificate[] getAcceptedIssuers() { return null; } 1156 | public void checkClientTrusted(X509Certificate[] arg0, String arg1) {} 1157 | public void checkServerTrusted(X509Certificate[] certs, String authType) throws CertificateException { 1158 | if (verbose) { 1159 | for (X509Certificate cert: certs) { 1160 | Util.logStderr("- Subject : " + cert.getSubjectX500Principal()); 1161 | Util.logStderr(" Serial # : " + String.join(":", cert.getSerialNumber().toString(16).split("(?<=\\G.{2})"))); 1162 | Util.logStderr(" Issued by: " + cert.getIssuerX500Principal()); 1163 | } 1164 | } 1165 | if (insecure) { 1166 | return; // no need to build the trustManager, just return (success) 1167 | } 1168 | try { 1169 | final X509TrustManager trustManager = getTrustManager(); 1170 | if (trustManager != null) { 1171 | trustManager.checkServerTrusted(certs, authType); 1172 | } 1173 | } catch (KeyStoreException e) { 1174 | try { 1175 | throw new CertificateException(e.getMessage()); 1176 | } catch (Exception ex) { 1177 | throw new RuntimeException(ex); 1178 | } 1179 | } 1180 | } 1181 | }}; 1182 | if (cert != null) { 1183 | Util.logStderr("Load key store"); 1184 | KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); // JKS for java, BKS for android 1185 | keyStore.load(cert.getInputStream(), password.toCharArray()); 1186 | cert.close(); 1187 | KeyManagerFactory factory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); 1188 | factory.init(keyStore, password.toCharArray()); 1189 | k_managers = factory.getKeyManagers(); 1190 | } 1191 | if (verbose) { 1192 | Util.logStderr("Create socket factory"); 1193 | } 1194 | SSLContext sc = SSLContext.getInstance("TLS"); 1195 | sc.init(k_managers, t_managers, new SecureRandom()); 1196 | return sc.getSocketFactory(); 1197 | } 1198 | 1199 | ///////////////////////////// Inner Classes & static instances /////////////////////////////////////// 1200 | 1201 | public interface Resolver { 1202 | T resolve(int httpCode, byte[] responseBody) throws Throwable; 1203 | } 1204 | 1205 | public static class ToStringResolver implements Resolver { 1206 | final private String charset; 1207 | public ToStringResolver(String charset) { this.charset = charset; } 1208 | @Override 1209 | public String resolve(int httpCode, byte[] raw) throws Throwable { return new String(raw, charset); } 1210 | } 1211 | 1212 | public static final Resolver RAW = new Resolver() { 1213 | @Override 1214 | public byte[] resolve(int httpCode, byte[] raw) { return raw; } 1215 | }; 1216 | 1217 | public static final ToStringResolver UTF8 = new ToStringResolver("UTF-8"); 1218 | public static final ToStringResolver GBK = new ToStringResolver("GBK"); 1219 | public static final ToStringResolver ISO_8859_1 = new ToStringResolver("ISO-8859-1"); 1220 | 1221 | public interface IO { 1222 | InputStream getInputStream(); 1223 | OutputStream getOutputStream(); 1224 | void setAppend(boolean append); 1225 | void close(); 1226 | } 1227 | 1228 | public static final class WrappedIO implements IO { 1229 | final InputStream is; 1230 | final OutputStream os; 1231 | public WrappedIO(String s, String charset) { this(Util.s2b(s, charset)); } 1232 | public WrappedIO(byte[] byteArray) { this(new ByteArrayInputStream(byteArray)); } 1233 | public WrappedIO(InputStream is) { this.is = is; this.os = null; } 1234 | public WrappedIO(OutputStream os) { this.is = null; this.os = os; } 1235 | public InputStream getInputStream() { return is; } 1236 | public OutputStream getOutputStream() { return os; } 1237 | public void setAppend(boolean append) {} // not supported 1238 | public void close() {} // wrapper is not responsible for closing 1239 | public String toString() { return "WrappedIO<" + is + "," + os + ">"; } 1240 | } 1241 | 1242 | @SuppressWarnings("ResultOfMethodCallIgnored") 1243 | public static final class FileIO implements IO { 1244 | private File f; 1245 | private transient InputStream is; 1246 | private transient OutputStream os; 1247 | boolean append = false; 1248 | 1249 | public FileIO(File f) { 1250 | this.f = f.getAbsoluteFile(); 1251 | } 1252 | 1253 | public FileIO(String path) { 1254 | this(new File(path)); 1255 | } 1256 | 1257 | public InputStream getInputStream() { 1258 | if (f.exists() && f.isFile() && f.canRead()) { 1259 | try { return is = new FileInputStream(f); } catch (Exception ignored) {} 1260 | } 1261 | return null; 1262 | } 1263 | 1264 | public OutputStream getOutputStream() { 1265 | Util.mkdirs(f.getParentFile()); 1266 | try { 1267 | f.createNewFile(); 1268 | f.setReadable(true, false); 1269 | f.setWritable(true, false); 1270 | os = new FileOutputStream(f, append); 1271 | } catch (Exception ignored) {} 1272 | return os; 1273 | } 1274 | 1275 | public void setAppend(boolean append) { 1276 | this.append = append; 1277 | } 1278 | 1279 | public void close() { 1280 | try { if (is != null) is.close(); } catch (Exception ignored) {} 1281 | try { if (os != null) os.close(); } catch (Exception ignored) {} 1282 | } 1283 | 1284 | public String toString() { 1285 | return "FileIO<" + f + ">"; 1286 | } 1287 | } 1288 | 1289 | public static final class MemIO extends ByteArrayOutputStream implements IO { 1290 | public MemIO() { super(0); } 1291 | public InputStream getInputStream() { return new ByteArrayInputStream(buf, 0, count); } 1292 | public OutputStream getOutputStream() { return this; } 1293 | public void setAppend(boolean append) { if (!append) this.reset(); } 1294 | public void close() {} // not needed 1295 | public String toString() { return "MemIO<" + this.hashCode() + ">"; } 1296 | /** 1297 | * This is useful when the MemIO was used as the target of --dump-header 1298 | * @return 1299 | */ 1300 | public Map parseDumpedHeader() { 1301 | Map result = new LinkedHashMap(); 1302 | String s = Util.b2s(this.toByteArray(), null, null); 1303 | for (String l: s.split("[\r\n]+")) { 1304 | if (l.trim().length() == 0) continue; 1305 | String[] kv = l.split(":", 2); 1306 | result.put(kv[0], kv.length > 1 ? kv[1].trim() : ""); 1307 | } 1308 | return result; 1309 | } 1310 | 1311 | public List parseCookieJar() { 1312 | return parseCookies(Util.b2s(this.toByteArray(), null, null)); 1313 | } 1314 | 1315 | } 1316 | 1317 | public static class CookieIO implements IO, java.net.CookieStore { 1318 | 1319 | public InputStream getInputStream() { throw new RuntimeException(); } 1320 | public OutputStream getOutputStream() { throw new RuntimeException(); } 1321 | public void setAppend(boolean append) {} 1322 | public void close() {} 1323 | 1324 | protected final Map> cookiesMap; 1325 | 1326 | public CookieIO() { 1327 | cookiesMap = new HashMap>(); 1328 | } 1329 | 1330 | protected Map> getCookiesMap() { 1331 | return cookiesMap; 1332 | } 1333 | 1334 | public void add(String uri, String key, String value) { 1335 | try { 1336 | this.add(new URI(uri), new HttpCookie(key, value)); 1337 | } catch (URISyntaxException e) { 1338 | throw new IllegalArgumentException(e); 1339 | } 1340 | } 1341 | 1342 | @Override 1343 | public void add(URI uri, HttpCookie cookie) { 1344 | normalize(uri, cookie); 1345 | Map> map = Util.mapListAdd(getCookiesMap(), ArrayList.class, cookie.getDomain()); 1346 | List cc = map.get(cookie.getDomain()); 1347 | cc.remove(cookie); 1348 | if (cookie.getMaxAge() != 0) cc.add(cookie); 1349 | } 1350 | 1351 | @Override 1352 | public List get(URI uri) { 1353 | List result = getCookies(); 1354 | String host = uri.getHost(); 1355 | for (ListIterator it = result.listIterator(); it.hasNext();) { 1356 | String domain = it.next().getDomain(); 1357 | if (!domainMatches(domain, host)) it.remove(); 1358 | } 1359 | return result; 1360 | } 1361 | 1362 | @Override 1363 | public List getCookies() { 1364 | List result = new ArrayList(); 1365 | for (List cc: getCookiesMap().values()) { 1366 | for (ListIterator it = cc.listIterator(); it.hasNext();) 1367 | if (it.next().hasExpired()) it.remove(); 1368 | result.addAll(cc); 1369 | } 1370 | return result; 1371 | } 1372 | 1373 | @Override 1374 | public List getURIs() { 1375 | Set result = new HashSet(); 1376 | for (HttpCookie cookie: getCookies()) { 1377 | String scheme = cookie.getSecure() ? "https" : "http"; 1378 | String domain = cookie.getDomain(); 1379 | if (domain.startsWith(".")) domain = domain.substring(1); 1380 | try { 1381 | result.add(new URI(scheme, domain, cookie.getPath(), null)); 1382 | } catch (URISyntaxException ignored) {} 1383 | } 1384 | return new ArrayList(result); 1385 | } 1386 | 1387 | @Override 1388 | public boolean remove(URI uri, HttpCookie cookie) { 1389 | normalize(uri, cookie); 1390 | List cc = getCookiesMap().get(cookie.getDomain()); 1391 | return cc != null && cc.remove(cookie); 1392 | } 1393 | 1394 | @Override 1395 | public boolean removeAll() { 1396 | getCookiesMap().clear(); 1397 | return true; 1398 | } 1399 | 1400 | private static void normalize(URI uri, HttpCookie cookie) { 1401 | if (cookie.getDomain() == null && uri != null) cookie.setDomain(uri.getHost()); 1402 | if (cookie.getPath() == null && uri != null) cookie.setPath(uri.getPath()); 1403 | if (Util.empty(cookie.getDomain())) throw new IllegalArgumentException("illegal cookie domain: " + cookie.getDomain()); 1404 | if (Util.empty(cookie.getPath())) cookie.setPath("/"); 1405 | cookie.setVersion(0); 1406 | } 1407 | 1408 | /** Check a string domain-matches a given domain string or not. Refer to section 5.1.3 RFC6265 */ 1409 | private static boolean domainMatches(String domain, String host) { 1410 | if (domain == null || host == null) return false; 1411 | if (domain.startsWith(".")) { // it's a suffix 1412 | return host.toLowerCase().endsWith(domain.toLowerCase()); 1413 | } else { 1414 | return host.equalsIgnoreCase(domain); 1415 | } 1416 | } 1417 | 1418 | } 1419 | 1420 | public static final class CookieStore extends CookieIO { 1421 | 1422 | private final ThreadLocal>> cookies = new ThreadLocal>>() { 1423 | @Override protected synchronized Map> initialValue() { 1424 | return new HashMap>(); 1425 | } 1426 | }; 1427 | 1428 | @Override 1429 | protected Map> getCookiesMap() { 1430 | return cookies.get(); 1431 | } 1432 | } 1433 | 1434 | public static final class Recoverable extends Exception { 1435 | private final int httpCode; 1436 | public Recoverable() { this(null, -1); } 1437 | public Recoverable(Throwable cause, int httpCode) { super(cause); this.httpCode = httpCode; } 1438 | } 1439 | 1440 | @SuppressWarnings({"WeakerAccess", "JavaDoc", "ConstantConditions", "ResultOfMethodCallIgnored", "StatementWithEmptyBody", "UnusedReturnValue", "SuspiciousMethodCalls"}) 1441 | final static class Util { 1442 | 1443 | public static boolean empty(String s) { 1444 | return s == null || s.length() == 0; 1445 | } 1446 | 1447 | public static List asList(Object o) { 1448 | if (o == null) return new ArrayList(0); 1449 | if (o instanceof Collection) { 1450 | return new ArrayList((Collection) o); 1451 | } else if (o.getClass().isArray()) { 1452 | ArrayList list = new ArrayList(); 1453 | for (int i = 0, n = Array.getLength(o); i < n; i++) list.add((T) Array.get(o, i)); 1454 | return list; 1455 | } else { 1456 | return listAdd(new ArrayList(1), (T) o); 1457 | } 1458 | } 1459 | 1460 | public static String qt(Object o) { 1461 | return o == null || o instanceof Boolean || o instanceof Number ? 1462 | "" + o : o instanceof Character ? "'" + o + "'" : "\"" + o + "\""; 1463 | } 1464 | 1465 | public static String dumpStackTrace(Throwable e, boolean singleLine) { 1466 | StringWriter sw = new StringWriter(); 1467 | e.printStackTrace(new PrintWriter(sw)); 1468 | String s = sw.toString(); 1469 | return singleLine ? s.replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t") : s; 1470 | } 1471 | 1472 | public static void logStderr(String msg, Object... args) { 1473 | if (args.length > 0) msg = String.format(msg, args); 1474 | DateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 1475 | System.err.println("[ERR] [" + fmt.format(new Date()) + "] " + msg); 1476 | //System.err.println("[ERR] [" + new Date() + "] " + msg); 1477 | } 1478 | 1479 | public static V mapGet(Map map, K key, V fallback) { 1480 | V v; 1481 | return map != null && (v = map.get(key)) != null ? v : fallback; 1482 | } 1483 | 1484 | public static Map> mapListAdd(Map> map, K key, V... val) { 1485 | return mapListAdd(map, ArrayList.class, key, val); 1486 | } 1487 | 1488 | public static > Map mapListAdd(Map map, Class collectionClass, K key, V... val) { 1489 | L l; 1490 | if ((l = map.get(key)) == null) try { 1491 | map.put(key, l = (L) collectionClass.newInstance()); 1492 | } catch (Exception ignored) { } 1493 | Collections.addAll(l, val); 1494 | return map; 1495 | } 1496 | 1497 | public static > V mapMapGet(Map map, K key, S subkey, V fallback) { 1498 | M m; 1499 | V ret; 1500 | return (m = map.get(key)) != null && (ret = m.get(subkey)) != null ? ret : fallback; 1501 | } 1502 | 1503 | public static Iterable safeIter(Iterable iter) { 1504 | return iter != null ? iter : new ArrayList(0); 1505 | } 1506 | 1507 | public static T[] safeArray(T[] array, Class componentType) { 1508 | return array != null ? array : (T[]) Array.newInstance(componentType, 0); 1509 | } 1510 | 1511 | public static Map newMap(Object... keyValuePairs) { 1512 | return mapPut(new LinkedHashMap(), keyValuePairs); 1513 | } 1514 | 1515 | public static > M mapPut(M map, Object... keyValuePairs) { 1516 | if ((keyValuePairs.length & 1) != 0) 1517 | throw new IllegalArgumentException("the number of keyValuePairs arguments must be odd"); 1518 | for (int i = 0, n = keyValuePairs.length; i < n; i += 2) { 1519 | map.put((K) keyValuePairs[i], (V) keyValuePairs[i + 1]); 1520 | } 1521 | return map; 1522 | } 1523 | 1524 | public static > L listAdd(L list, T... values) { 1525 | list.addAll(Arrays.asList(values)); 1526 | return list; 1527 | } 1528 | 1529 | public static class Ref { 1530 | public int i; 1531 | public T v; 1532 | 1533 | public Ref(T v) { 1534 | this(0, v); 1535 | } 1536 | 1537 | public Ref(int i, T v) { 1538 | setInt(i); 1539 | set(v); 1540 | } 1541 | 1542 | public T get() { 1543 | return v; 1544 | } 1545 | 1546 | public void set(T v) { 1547 | this.v = v; 1548 | } 1549 | 1550 | public int getInt() { 1551 | return i; 1552 | } 1553 | 1554 | public void setInt(int i) { 1555 | this.i = i; 1556 | } 1557 | 1558 | @Override 1559 | public boolean equals(Object obj) { 1560 | if (!(obj instanceof Ref)) return false; 1561 | Ref o; 1562 | return (o = (Ref) obj) != null && i == o.i && (v == null ? o.v == null : v.equals(o.v)); 1563 | } 1564 | 1565 | @Override 1566 | public int hashCode() { 1567 | return i + (v == null ? 0 : v.hashCode()); 1568 | } 1569 | 1570 | @Override 1571 | public String toString() { 1572 | return String.format("Ref{%s, %s}", i, qt(v)); 1573 | } 1574 | 1575 | } 1576 | 1577 | public static String urlDecode(String s, String enc) { 1578 | if (!empty(s)) try { 1579 | return URLDecoder.decode(s, enc); 1580 | } catch (Exception ignored) { } 1581 | return s; 1582 | } 1583 | 1584 | public static String b2s(byte[] bb, String charset, String fallback) { 1585 | return b2s(bb, 0, bb.length, charset, fallback); 1586 | } 1587 | 1588 | public static String b2s(byte[] bb, int offset, int count, String charset, String fallback) { 1589 | try { 1590 | int start = bb.length - offset >= 3 && bb[offset] == 0xEF && bb[offset + 1] == 0xBB && bb[offset + 2] == 0xBF ? 3 : 0; // deal BOM 1591 | return new String(bb, offset + start, count - start, charset == null ? "UTF-8" : charset); 1592 | } catch (Exception e) { 1593 | return fallback; 1594 | } 1595 | } 1596 | 1597 | public static byte[] s2b(String s, String charset) { 1598 | try { 1599 | return s.getBytes(charset == null ? "UTF-8" : charset); 1600 | } catch (Exception e) { 1601 | return null; 1602 | } 1603 | } 1604 | 1605 | public static String[][] split(String s, String delim1, String delim2) { 1606 | String[] ss = s.split(delim1); 1607 | String[][] result = new String[ss.length][]; 1608 | for (int i = ss.length; --i >= 0; result[i] = ss[i].split(delim2)); 1609 | return result; 1610 | } 1611 | 1612 | public static Map split(String s, String entryDelim, String kvDelim, Map toMap) { 1613 | String[] ss = s.split(entryDelim); 1614 | if (toMap == null) toMap = new HashMap(ss.length); 1615 | for (String l : ss) { 1616 | String[] sub = l.split(kvDelim); 1617 | toMap.put(sub[0].trim(), sub.length > 1 ? sub[1].trim() : ""); 1618 | } 1619 | return toMap; 1620 | } 1621 | 1622 | public static String join(Object mapOrColl, String delim, String subDelim) { 1623 | List> all = new ArrayList>(); 1624 | if (mapOrColl == null) { // do nothing 1625 | } else if (mapOrColl instanceof Map) { 1626 | for (Map.Entry kv : ((Map) mapOrColl).entrySet()) { 1627 | all.add(listAdd(new ArrayList(2), kv.getKey(), kv.getValue())); 1628 | } 1629 | } else if (mapOrColl instanceof Collection) { 1630 | for (Object o : (Collection) mapOrColl) all.add(asList(o)); 1631 | } else if (mapOrColl.getClass().isArray()) { 1632 | for (int i = 0, n = Array.getLength(mapOrColl); i < n; all.add(asList(Array.get(mapOrColl, i++)))) 1633 | ; 1634 | } else { // plain object 1635 | all.add(asList(mapOrColl)); 1636 | } 1637 | StringBuilder sb = new StringBuilder(); 1638 | int i = 0, j; 1639 | for (List sub : all) { 1640 | if (i++ > 0) sb.append(delim); 1641 | j = 0; 1642 | for (Object o : sub) sb.append(j++ > 0 ? subDelim : "").append(o); 1643 | } 1644 | return sb.toString(); 1645 | } 1646 | 1647 | public static String base64Encode(byte[] bb) { 1648 | Class clz = getClass("java.util.Base64", null); 1649 | if (clz != null) { 1650 | Object encoder = invokeSilent(null, clz, "getEncoder", false, null); 1651 | return (String) invokeSilent(encoder, null, "encodeToString", false, "[B", (Object) bb); 1652 | } 1653 | clz = getClass("sun.misc.BASE64Encoder", null); 1654 | if (clz != null) { 1655 | Object encoder = createInstance(clz, "", true); 1656 | return ((String) invokeSilent(encoder, null, "encode", true, "[B", (Object) bb)).replaceAll("[\r\n]+", ""); 1657 | } 1658 | clz = getClass("org.apache.commons.codec.binary.Base64", null); 1659 | if (clz != null) { 1660 | return (String) invokeSilent(null, clz, "encodeBase64String", false, "[B", (Object) bb); 1661 | } 1662 | clz = getClass("android.util.Base64", null); 1663 | if (clz != null) { 1664 | return (String) invokeSilent(null, clz, "encodeToString", false, "[BI", bb, 2); // NO_WRAP 1665 | } 1666 | throw new RuntimeException(new NoSuchMethodException("base64Encode")); 1667 | } 1668 | 1669 | public static byte[] readStream(InputStream is, boolean close) { 1670 | return readStream(is, 0, close); 1671 | } 1672 | 1673 | public static byte[] readStream(InputStream is, int interruptOnSize, boolean close) { 1674 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 1675 | int count = 0, c; 1676 | while ((c = pipeStream(is, bos)) > 0 && (interruptOnSize <= 0 || count < interruptOnSize)) count += c; 1677 | if (c < 0) count += (c & PIPE_COUNT_MASK); 1678 | byte[] result = c < 0 && count == 0 ? null : bos.toByteArray(); 1679 | if (close) try { 1680 | is.close(); 1681 | } catch (Exception ignored) { 1682 | } 1683 | return result; 1684 | } 1685 | 1686 | public static final int PIPE_COUNT_MASK = 0x7FFFFFFF; 1687 | 1688 | private static final int BUFFER_SIZE = 10000; 1689 | 1690 | public static int pipeStream(InputStream source, OutputStream destination) { 1691 | byte[] bb = new byte[BUFFER_SIZE]; 1692 | int len, count = 0; 1693 | do { 1694 | len = 0; 1695 | try { 1696 | len = source.read(bb); 1697 | } catch (SocketTimeoutException e) { // no data, but the socket connection is still alive 1698 | } catch (SocketException e) { // EOF or socket disconnected 1699 | len = -1; 1700 | } catch (IOException e) { // unexpected exceptions 1701 | throw new RuntimeException(e); 1702 | } 1703 | if (len > 0) { 1704 | try { 1705 | destination.write(bb, 0, len); 1706 | } catch (IOException e) { // unexpected exceptions while writing 1707 | throw new RuntimeException(e); 1708 | } 1709 | count += len; 1710 | } 1711 | } while (len == BUFFER_SIZE); 1712 | return len < 0 ? (0x80000000 | count) : count; // len < 0 -> EOF reached 1713 | } 1714 | 1715 | public static void mkdirs(File dir) { 1716 | File parent = dir.getAbsoluteFile(); 1717 | List mkdir = new ArrayList(); 1718 | for (; !parent.exists() || !parent.isDirectory(); parent = parent.getParentFile()) { 1719 | mkdir.add(parent); 1720 | } 1721 | for (int i = mkdir.size(); --i >= 0; ) { 1722 | File d = mkdir.get(i); 1723 | d.mkdir(); 1724 | d.setReadable(true, false); 1725 | d.setWritable(true, false); 1726 | } 1727 | } 1728 | 1729 | public static Class getClass(String className, ClassLoader cl) { 1730 | try { 1731 | return (cl != null ? cl : CUrl.class.getClassLoader()).loadClass(className); 1732 | } catch (ClassNotFoundException e) { 1733 | return null; 1734 | } 1735 | } 1736 | 1737 | public static T createInstance(Class cls, String signature, boolean ignoreAccess, Object... args) { 1738 | if (signature == null && args.length == 0) { 1739 | try { 1740 | return cls.newInstance(); 1741 | } catch (Exception ex) { 1742 | throw new IllegalArgumentException(ex); 1743 | } 1744 | } 1745 | return (T) invoke(null, cls, "", ignoreAccess, signature, args); 1746 | } 1747 | 1748 | public static Object getField(Object thiz, Class cls, String fieldName, Object fallback, boolean ignoreAccess) { 1749 | if (thiz == null && cls == null || fieldName == null) 1750 | throw new NullPointerException("inst=" + thiz + ",class=" + cls + ",field=" + fieldName); 1751 | try { 1752 | for (MemberInfo mi : safeIter(getMembers(thiz != null ? thiz.getClass() : cls, fieldName))) { 1753 | if (-1 == mi.numArgs && (ignoreAccess || (mi.member.getModifiers() & Modifier.PUBLIC) != 0)) { 1754 | AccessibleObject acc; 1755 | if (ignoreAccess && !(acc = (AccessibleObject) mi.member).isAccessible()) acc.setAccessible(true); 1756 | return ((Field) mi.member).get(thiz); 1757 | } 1758 | } 1759 | } catch (Exception ignored) { 1760 | } 1761 | return fallback; 1762 | } 1763 | 1764 | public static Object invokeSilent(Object thiz, Class cls, String methodName, boolean ignoreAccess, String signature, Object... args) { 1765 | try { 1766 | return invoke(thiz, cls, methodName, ignoreAccess, signature, args); 1767 | } catch (Exception ignored) { 1768 | } 1769 | return null; 1770 | } 1771 | 1772 | public static Object invoke(Object thiz, Class cls, String methodName, boolean ignoreAccess, String signature, Object... args) { 1773 | if (thiz == null && cls == null || methodName == null) 1774 | throw new NullPointerException("inst=" + thiz + ",class=" + cls + ",method=" + methodName); 1775 | List found = getMembers(thiz != null ? thiz.getClass() : cls, methodName); 1776 | try { 1777 | Member m = null; 1778 | if (found == null) { // do nothing 1779 | } else if (signature == null) { 1780 | int len = args.length; 1781 | for (MemberInfo mi : found) { 1782 | if (len == mi.numArgs && (ignoreAccess || (mi.member.getModifiers() & Modifier.PUBLIC) != 0)) { 1783 | m = mi.member; 1784 | break; 1785 | } 1786 | } 1787 | } else { 1788 | signature = signature.replace('/', '.'); 1789 | for (MemberInfo mi : found) { 1790 | if (signature.equals(mi.signature) && (ignoreAccess || (mi.member.getModifiers() & Modifier.PUBLIC) != 0)) { 1791 | m = mi.member; 1792 | break; 1793 | } 1794 | } 1795 | } 1796 | if (m == null) { 1797 | StringBuilder msg = new StringBuilder().append('"').append(methodName).append('"'); 1798 | if (signature == null) { 1799 | msg.append(" with ").append(args.length).append(" parameter(s)"); 1800 | } else { 1801 | msg.append(" with signature \"").append(signature).append("\""); 1802 | } 1803 | throw new NoSuchMethodException(msg.toString()); 1804 | } 1805 | AccessibleObject acc; 1806 | if (ignoreAccess && !(acc = (AccessibleObject) m).isAccessible()) acc.setAccessible(true); 1807 | return m instanceof Method ? ((Method) m).invoke(thiz, args) : ((Constructor) m).newInstance(args); 1808 | } catch (Exception ex) { 1809 | throw new IllegalArgumentException(ex); 1810 | } 1811 | } 1812 | 1813 | private static final Map primaryTypes = newMap( 1814 | byte.class, 'B', 1815 | char.class, 'C', 1816 | double.class, 'D', 1817 | float.class, 'F', 1818 | int.class, 'I', 1819 | long.class, 'J', 1820 | short.class, 'S', 1821 | void.class, 'V', 1822 | boolean.class, 'Z'); 1823 | 1824 | public static String getSignature(Class... types) { 1825 | StringBuilder sb = new StringBuilder(); 1826 | for (Class t : types) { 1827 | while (t.isArray()) { 1828 | sb.append('['); 1829 | t = t.getComponentType(); 1830 | } 1831 | Character c; 1832 | if ((c = (Character) primaryTypes.get(t)) != null) { 1833 | sb.append(c); 1834 | } else { 1835 | sb.append('L').append(t.getName()).append(';'); 1836 | } 1837 | } 1838 | return sb.toString(); 1839 | } 1840 | 1841 | private static final Map, Map>> mapClassMembers = new HashMap, Map>>(); 1842 | 1843 | private static synchronized List getMembers(Class cls, String name) { 1844 | if (!mapClassMembers.containsKey(cls)) { 1845 | Map> map; 1846 | mapClassMembers.put(cls, map = new LinkedHashMap>()); 1847 | Class clss = cls; 1848 | while (clss != null && !Object.class.equals(clss)) { 1849 | for (Constructor c : safeArray(clss.getDeclaredConstructors(), Constructor.class)) { 1850 | Class[] ptypes = c.getParameterTypes(); 1851 | mapListAdd(map, "", new MemberInfo(getSignature(ptypes), ptypes.length, c)); 1852 | } 1853 | for (Method m : safeArray(clss.getDeclaredMethods(), Method.class)) { 1854 | Class[] ptypes = m.getParameterTypes(); 1855 | mapListAdd(map, m.getName(), new MemberInfo(getSignature(ptypes), ptypes.length, m)); 1856 | } 1857 | for (Field f : safeArray(clss.getDeclaredFields(), Field.class)) { 1858 | mapListAdd(map, f.getName(), new MemberInfo(null, -1, f)); 1859 | } 1860 | clss = clss.getSuperclass(); 1861 | } 1862 | } 1863 | return mapMapGet(mapClassMembers, cls, name, null); 1864 | } 1865 | 1866 | private static class MemberInfo { 1867 | String signature; // null for field 1868 | int numArgs; // -1 for field 1869 | Member member; 1870 | 1871 | MemberInfo(String sign, int num, Member member) { 1872 | signature = sign; 1873 | numArgs = num; 1874 | this.member = member; 1875 | } 1876 | 1877 | public final String toString() { 1878 | return member.toString(); 1879 | } 1880 | } 1881 | 1882 | } 1883 | 1884 | public static void main(String[] args) { 1885 | System.out.println(new CUrl().opt(args).exec(null)); 1886 | } 1887 | 1888 | } 1889 | 1890 | 1891 | -------------------------------------------------------------------------------- /src/test/java/com/roxstudio/utils/CUrlTest.java: -------------------------------------------------------------------------------- 1 | package com.roxstudio.utils; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.jsoup.Jsoup; 5 | import org.jsoup.nodes.Document; 6 | import org.junit.Test; 7 | 8 | import java.nio.charset.StandardCharsets; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.concurrent.CountDownLatch; 13 | 14 | import static org.junit.Assert.assertEquals; 15 | import static org.junit.Assert.assertTrue; 16 | 17 | public class CUrlTest { 18 | 19 | private static final boolean ENABLE_FIDDLER_FOR_ALL_TEST = true; 20 | 21 | @Test 22 | public void gzippedResponse() { 23 | CUrl curl = curl("http://httpbin.org/gzip") 24 | .opt("--compressed"); // Suggest the server using gzipped response 25 | curl.exec(); 26 | assertEquals(curl.getHttpCode(), 200); 27 | } 28 | 29 | @Test 30 | public void headMethod() { 31 | CUrl curl = curl("http://httpbin.org/get") 32 | .dumpHeader("-") // output to stdout 33 | .opt("--head"); 34 | curl.exec(); 35 | assertEquals(curl.getHttpCode(), 200); 36 | } 37 | 38 | @Test 39 | public void getRedirectLocation() { 40 | CUrl curl = curl("http://httpbin.org/redirect-to") 41 | .data("url=http://www.baidu.com", "UTF-8") // do URL-Encoding for each form value 42 | .opt("--get"); // force using GET method 43 | curl.exec(); 44 | String location = null; 45 | List responseHeaders = curl.getResponseHeaders().get(0); // Follow redirect is disabled, only one response here 46 | for (String[] headerValuePair: responseHeaders) { 47 | if ("Location".equals(headerValuePair[0])) { 48 | location = headerValuePair[1]; 49 | break; 50 | } 51 | } 52 | assertEquals(302, curl.getHttpCode()); 53 | assertEquals(location, "http://www.baidu.com"); 54 | } 55 | 56 | @Test 57 | public void insecureHttpsViaFiddler() { 58 | CUrl curl = curl("https://httpbin.org/get") 59 | .proxy("127.0.0.1", 8888) // Use Fiddler to capture & parse HTTPS traffic 60 | .insecure(); // Ignore certificate check since it's issued by Fiddler 61 | curl.exec(); 62 | assertEquals(200, curl.getHttpCode()); 63 | } 64 | 65 | @Test 66 | public void httpPost() { 67 | CUrl curl = curl("http://httpbin.org/post") 68 | .data("hello=world&foo=bar") 69 | .data("foo=overwrite"); 70 | curl.exec(); 71 | assertEquals(200, curl.getHttpCode()); 72 | } 73 | 74 | @Test 75 | public void uploadMultipleFiles() { 76 | CUrl.MemIO inMemFile = new CUrl.MemIO(); 77 | try { inMemFile.getOutputStream().write("text file content blabla...".getBytes()); } catch (Exception ignored) {} 78 | CUrl curl = curl("http://httpbin.org/post") 79 | .form("formItem", "value") // a plain form item 80 | .form("file", inMemFile) // in-memory "file" 81 | .form("image", new CUrl.FileIO("src/test/resources/a2.png")); // A file in storage, change it to an existing path to avoid failure 82 | curl.exec(); 83 | assertEquals(200, curl.getHttpCode()); 84 | } 85 | 86 | @Test 87 | public void httpBasicAuth() { 88 | CUrl curl = curl("http://httpbin.org/basic-auth/abc/aaa") 89 | .proxy("127.0.0.1", 8888) 90 | .opt("-u", "abc:aaa"); 91 | curl.exec(); 92 | assertEquals(200, curl.getHttpCode()); 93 | } 94 | 95 | @Test 96 | public void customUserAgentAndHeaders() { 97 | String mobileUserAgent = "Mozilla/5.0 (Linux; U; Android 8.0.0; zh-cn; KNT-AL10 Build/HUAWEIKNT-AL10) AppleWebKit/537.36 (KHTML, like Gecko) MQQBrowser/7.3 Chrome/37.0.0.0 Mobile Safari/537.36"; 98 | Map fakeAjaxHeaders = new HashMap(); 99 | fakeAjaxHeaders.put("X-Requested-With", "XMLHttpRequest"); 100 | fakeAjaxHeaders.put("Referer", "http://somesite.com/fake_referer"); 101 | CUrl curl = curl("http://httpbin.org/get") 102 | .opt("-A", mobileUserAgent) // simulate a mobile browser 103 | .headers(fakeAjaxHeaders) // simulate an AJAX request 104 | .header("X-Auth-Token: xxxxxxx"); // other custom header, this might be calculated elsewhere 105 | curl.exec(); 106 | assertEquals(200, curl.getHttpCode()); 107 | } 108 | 109 | @SuppressWarnings("unchecked") 110 | @Test 111 | public void customResolver() { 112 | CUrl curl = curl("http://httpbin.org/json"); 113 | // execute request and convert response to JSON 114 | Map json = curl.exec(jsonResolver, null); 115 | assertEquals(200, curl.getHttpCode()); 116 | assertEquals("Yours Truly", deepGet(json, "slideshow.author")); 117 | assertEquals("Why WonderWidgets are great", deepGet(json, "slideshow.slides.1.items.0")); 118 | // execute request and convert response to HTML 119 | curl = curl("http://httpbin.org/html"); 120 | Document html = curl.exec(htmlResolver, null); 121 | assertEquals(200, curl.getHttpCode()); 122 | assertEquals("Herman Melville - Moby-Dick", html.select("h1:first-child").text()); 123 | } 124 | 125 | @Test 126 | public void threadSafeCookies() { 127 | final CountDownLatch count = new CountDownLatch(3); 128 | final CUrl[] curls = new CUrl[3]; 129 | for (int i = 3; --i >= 0;) { 130 | final int idx = i; 131 | new Thread() { 132 | public void run() { 133 | CUrl curl = curls[idx] = curl("http://httpbin.org/get") 134 | .cookie("thread" + idx + "=#" + idx); 135 | curl.exec(); 136 | count.countDown(); 137 | } 138 | }.start(); 139 | } 140 | try { count.await(); } catch (Exception ignored) {} // make sure all requests are done 141 | assertEquals(200, curls[0].getHttpCode()); 142 | assertEquals("thread0=#0", deepGet(curls[0].getStdout(jsonResolver, null), "headers.Cookie")); 143 | assertEquals("thread1=#1", deepGet(curls[1].getStdout(jsonResolver, null), "headers.Cookie")); 144 | assertEquals("thread2=#2", deepGet(curls[2].getStdout(jsonResolver, null), "headers.Cookie")); 145 | } 146 | 147 | @Test 148 | public void reuseCookieAcrossThreads() { 149 | final CUrl.IO cookieJar = new CUrl.MemIO(); 150 | final CountDownLatch lock = new CountDownLatch(1); 151 | new Thread() { 152 | public void run() { 153 | CUrl curl = curl("http://httpbin.org/cookies/set/from/server") // server-side Set-Cookie response header 154 | .cookie("foo=bar; hello=world") // multiple cookies are seperated by "; " 155 | .cookieJar(cookieJar); // write cookies to an IO instance 156 | curl.exec(); 157 | lock.countDown(); 158 | } 159 | }.start(); 160 | try { lock.await(); } catch (Exception ignored) {} // make sure request is done 161 | CUrl curl = curl("http://httpbin.org/cookies") 162 | .cookie(cookieJar); // reuse cookies 163 | curl.exec(); 164 | assertEquals(200, curl.getHttpCode()); 165 | assertEquals("bar", deepGet(curl.getStdout(jsonResolver, null), "cookies.foo")); 166 | assertEquals("world", deepGet(curl.getStdout(jsonResolver, null), "cookies.hello")); 167 | assertEquals("server", deepGet(curl.getStdout(jsonResolver, null), "cookies.from")); 168 | } 169 | 170 | @Test 171 | public void selfSignedCertificate() { 172 | CUrl curl = new CUrl("https://www.baidu.com/") 173 | .opt("--verbose") 174 | .cert(new CUrl.FileIO("D:/tmp/test_jks.jks"), "123456") 175 | .proxy("127.0.0.1", 8888); 176 | System.out.println(curl.exec(CUrl.UTF8, null)); 177 | } 178 | 179 | @Test 180 | public void givenCorrectRootCA_whenGet_thenSuccess() { 181 | // If you have a corporate firewall that intercepts outbound TLS connections, you will need to provide your own 182 | // root jks bundle 183 | // keytool -importcert -file inputfile.pem -keystore output_file.jks -storetype jks 184 | CUrl curl = new CUrl("https://www.baidu.com/") 185 | .opt("--verbose") 186 | .opt("--cacert", "src/test/resources/global_sign_root_r1.jks") 187 | .proxy("127.0.0.1", 8888); 188 | curl.exec(); 189 | assertEquals(200, curl.getHttpCode()); 190 | } 191 | 192 | @Test 193 | public void givenWrongRootCA_whenGet_thenException() { 194 | // any random root CA that will fail all targets 195 | CUrl curl = new CUrl("https://www.baidu.com/") 196 | .opt("--verbose") 197 | .opt("--cacert", "src/test/resources/random_root.jks") 198 | .proxy("127.0.0.1", 8888); 199 | String result = new String(curl.exec(), StandardCharsets.UTF_8); 200 | assertTrue(result.contains("unable to find valid certification path to requested target")); 201 | } 202 | 203 | /////////////////////////////////////////////////////////////////////////////// 204 | 205 | private CUrl curl(String url) { 206 | CUrl curl = new CUrl(url); 207 | if (ENABLE_FIDDLER_FOR_ALL_TEST) { 208 | curl.proxy("127.0.0.1", 8888).insecure(); 209 | } 210 | return curl; 211 | } 212 | 213 | /** Implement a custom resolver that convert raw response to JSON */ 214 | private CUrl.Resolver> jsonResolver = new CUrl.Resolver>() { 215 | @SuppressWarnings("unchecked") 216 | @Override 217 | public Map resolve(int httpCode, byte[] responseBody) throws Throwable { 218 | String json = new String(responseBody, "UTF-8"); 219 | return new ObjectMapper().readValue(json, Map.class); 220 | } 221 | }; 222 | /** Implement a custom resolver that convert raw response to Jsoup Document */ 223 | private CUrl.Resolver htmlResolver = new CUrl.Resolver() { 224 | @SuppressWarnings("unchecked") 225 | @Override 226 | public Document resolve(int httpCode, byte[] responseBody) throws Throwable { 227 | String html = new String(responseBody, "UTF-8"); 228 | return Jsoup.parse(html); 229 | } 230 | }; 231 | 232 | @SuppressWarnings("unchecked") 233 | private static T deepGet(Object obj, String names) { 234 | boolean isMap; 235 | if (!(isMap = obj instanceof Map) && !(obj instanceof List)) return null; 236 | int idx = names.indexOf('.'); 237 | String n = idx > 0 ? names.substring(0, idx) : names; 238 | names = idx > 0 ? names.substring(idx + 1) : null; 239 | if (isMap) { 240 | obj = ((Map) obj).get(n); 241 | } else { 242 | idx = Integer.parseInt(n); 243 | if (idx < 0 || idx >= ((List) obj).size()) return null; 244 | obj = ((List) obj).get(idx); 245 | } 246 | return names != null ? CUrlTest.deepGet(obj, names) : obj != null ? (T) obj : null; 247 | } 248 | 249 | } 250 | -------------------------------------------------------------------------------- /src/test/resources/a2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rockswang/java-curl/62dcc1889df2f616bce72c2f350b4cc3027b84d6/src/test/resources/a2.png -------------------------------------------------------------------------------- /src/test/resources/global_sign_root_r1.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rockswang/java-curl/62dcc1889df2f616bce72c2f350b4cc3027b84d6/src/test/resources/global_sign_root_r1.jks -------------------------------------------------------------------------------- /src/test/resources/random_root.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rockswang/java-curl/62dcc1889df2f616bce72c2f350b4cc3027b84d6/src/test/resources/random_root.jks --------------------------------------------------------------------------------