├── .github ├── workflows │ ├── requirements.txt │ └── test.yml └── dependabot.yml ├── AUTHORS ├── REUSE.toml ├── LICENSE ├── LICENSES └── curl.txt ├── wcurl.1 ├── wcurl.md ├── CHANGELOG.md ├── README.md ├── tests └── tests.sh └── wcurl /.github/workflows/requirements.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) Viktor Szakats. See LICENSE.md 2 | # SPDX-License-Identifier: curl 3 | 4 | codespell==2.4.1 5 | reuse==6.1.2 6 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | 6 | Arthur Diniz 7 | Ben Zanin 8 | Daniel Stenberg 9 | Guilherme Puida Moreira 10 | Patrick Stoeckle 11 | Ryan Carsten Schmidt 12 | Samuel Henrique 13 | Sergio Durigan Junior 14 | Viktor Szakats 15 | WGH 16 | XhmikosR 17 | Xi Ruoyao 18 | -------------------------------------------------------------------------------- /REUSE.toml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: curl 2 | # SPDX-FileCopyrightText: Daniel Stenberg, , et al. 3 | 4 | # This file describes the licensing and copyright situation for files that 5 | # cannot be annotated directly, for example because of being simply 6 | # uncommentable. Unless this is the case, a file should be annotated directly. 7 | # 8 | # This follows the REUSE specification: https://reuse.software/spec-3.2/#reusetoml 9 | 10 | version = 1 11 | SPDX-PackageName = "wcurl" 12 | SPDX-PackageDownloadLocation = "https://curl.se/wcurl/" 13 | 14 | [[annotations]] 15 | path = [ 16 | "wcurl.1", 17 | ] 18 | SPDX-FileCopyrightText = "Copyright (C) Samuel Henrique , Sergio Durigan Junior and many contributors, see the AUTHORS file." 19 | SPDX-License-Identifier = "curl" 20 | # If there is licensing/copyright information in or next to these files, prefer that 21 | precedence = "closest" 22 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Copyright (C) Viktor Szakats. See LICENSE.md 2 | # SPDX-License-Identifier: curl 3 | 4 | # https://docs.github.com/code-security/dependabot/working-with-dependabot/dependabot-options-reference 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: 'github-actions' 9 | directory: '/' 10 | schedule: 11 | interval: 'quarterly' 12 | cooldown: 13 | default-days: 30 14 | groups: 15 | gha-dependencies: 16 | patterns: 17 | - '*' 18 | commit-message: 19 | prefix: 'GHA:' 20 | 21 | - package-ecosystem: 'pip' 22 | directory: '.github/workflows' 23 | schedule: 24 | interval: 'quarterly' 25 | cooldown: 26 | default-days: 7 27 | semver-major-days: 15 28 | semver-minor-days: 7 29 | semver-patch-days: 3 30 | groups: 31 | pip-dependencies: 32 | patterns: 33 | - '*' 34 | commit-message: 35 | prefix: 'GHA:' 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | COPYRIGHT AND PERMISSION NOTICE 2 | 3 | Copyright (C) Samuel Henrique , Sergio Durigan 4 | Junior and many contributors, see the AUTHORS 5 | file. 6 | 7 | All rights reserved. 8 | 9 | Permission to use, copy, modify, and distribute this software for any purpose 10 | with or without fee is hereby granted, provided that the above copyright 11 | notice and this permission notice appear in all copies. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN 16 | NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | OR OTHER DEALINGS IN THE SOFTWARE. 20 | 21 | Except as contained in this notice, the name of a copyright holder shall not 22 | be used in advertising or otherwise to promote the sale, use or other dealings 23 | in this Software without prior written authorization of the copyright holder. 24 | -------------------------------------------------------------------------------- /LICENSES/curl.txt: -------------------------------------------------------------------------------- 1 | COPYRIGHT AND PERMISSION NOTICE 2 | 3 | Copyright (C) Samuel Henrique , Sergio Durigan 4 | Junior and many contributors, see the AUTHORS 5 | file. 6 | 7 | All rights reserved. 8 | 9 | Permission to use, copy, modify, and distribute this software for any purpose 10 | with or without fee is hereby granted, provided that the above copyright 11 | notice and this permission notice appear in all copies. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN 16 | NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | OR OTHER DEALINGS IN THE SOFTWARE. 20 | 21 | Except as contained in this notice, the name of a copyright holder shall not 22 | be used in advertising or otherwise to promote the sale, use or other dealings 23 | in this Software without prior written authorization of the copyright holder. 24 | -------------------------------------------------------------------------------- /wcurl.1: -------------------------------------------------------------------------------- 1 | .\" generated by cd2nroff 0.1 from wcurl.md 2 | .TH wcurl 1 "2025-11-09" wcurl 3 | .SH NAME 4 | \fBwcurl\fP \- a simple wrapper around curl to easily download files. 5 | .SH SYNOPSIS 6 | \fBwcurl ...\fP 7 | 8 | \fBwcurl [\--curl\-options ]... [\--dry\-run] [\--no\-decode\-filename] [\-o|\-O|\--output ] [\--] ...\fP 9 | 10 | \fBwcurl [\--curl\-options=]... [\--dry\-run] [\--no\-decode\-filename] [\--output=] [\--] ...\fP 11 | 12 | \fBwcurl \-V|\--version\fP 13 | 14 | \fBwcurl \-h|\--help\fP 15 | .SH DESCRIPTION 16 | \fBwcurl\fP is a simple curl wrapper which lets you use curl to download files 17 | without having to remember any parameters. 18 | 19 | Simply call \fBwcurl\fP with a list of URLs you want to download and \fBwcurl\fP 20 | picks sane defaults. 21 | 22 | If you need anything more complex, you can provide any of curl\(aqs supported 23 | parameters via the \fB\--curl\-options\fP option. Just beware that you likely 24 | should be using curl directly if your use case is not covered. 25 | 26 | By default, \fBwcurl\fP does: 27 | .IP "* Percent-encode whitespace in URLs;" 28 | .IP "* Download multiple URLs in parallel" 29 | .nf 30 | if the installed curl's version is \>= 7.66.0 (--parallel); 31 | .fi 32 | .IP "* Use a total number of 5 parallel connections to the same protocol + hostname + port number target" 33 | .nf 34 | if the installed curl's version is \>= 8.16.0 (--parallel-max-host); 35 | .fi 36 | .IP "* Follow redirects;" 37 | .IP "* Automatically choose a filename as output;" 38 | .IP "* Avoid overwriting files" 39 | .nf 40 | if the installed curl's version is \>= 7.83.0 (--no-clobber); 41 | .fi 42 | .IP "* Perform retries;" 43 | .IP "* Set the downloaded file timestamp" 44 | .nf 45 | to the value provided by the server, if available; 46 | .fi 47 | .IP "* Default to https" 48 | .nf 49 | if the URL does not contain any scheme; 50 | .fi 51 | .IP "* Disable curl's URL globbing parser" 52 | .nf 53 | so {} and [] characters in URLs are not treated specially; 54 | .fi 55 | .IP "* Percent-decode the resulting filename;" 56 | .IP "* Use 'index.html' as the default filename" 57 | .nf 58 | if there is none in the URL. 59 | .fi 60 | .SH OPTIONS 61 | .IP "--curl-options, --curl-options=\..." 62 | Specify extra options to be passed when invoking curl. May be specified more 63 | than once. 64 | .IP "-o, -O, --output, --output=\" 65 | Use the provided output path instead of getting it from the URL. If multiple 66 | URLs are provided, resulting files share the same name with a number appended to 67 | the end (curl >= 7.83.0). If this option is provided multiple times, only the 68 | last value is considered. 69 | .IP --no-decode-filename 70 | Do not percent\-decode the output filename, even if the percent\-encoding in the 71 | URL was done by \fBwcurl\fP, e.g.: The URL contained whitespace. 72 | .IP --dry-run 73 | Do not actually execute curl, just print what would be invoked. 74 | .IP "-V, \--version" 75 | Print version information. 76 | .IP "-h, \--help" 77 | Print help message. 78 | .SH CURL_OPTIONS 79 | Any option supported by curl can be set here. This is not used by \fBwcurl\fP; it 80 | is instead forwarded to the curl invocation. 81 | .SH URL 82 | URL to be downloaded. Anything that is not a parameter is considered 83 | an URL. Whitespace is percent\-encoded and the URL is passed to curl, which 84 | then performs the parsing. May be specified more than once. 85 | .SH EXAMPLES 86 | Download a single file: 87 | 88 | \fBwcurl example.com/filename.txt\fP 89 | 90 | Download two files in parallel: 91 | 92 | \fBwcurl example.com/filename1.txt example.com/filename2.txt\fP 93 | 94 | Download a file passing the \fB\--progress\-bar\fP and \fB\--http2\fP flags to curl: 95 | 96 | \fBwcurl \--curl\-options="\--progress\-bar \--http2" example.com/filename.txt\fP 97 | 98 | * Resume from an interrupted download. The options necessary to resume the download (\fI\--clobber \--continue\-at \-\fP) must be the \fBlast\fP options specified in \fI\--curl\-options\fP. Note that the only way to resume interrupted downloads is to allow wcurl to overwrite the destination file: 99 | 100 | \fBwcurl \--curl\-options="\--clobber \--continue\-at \-" example.com/filename.txt\fP 101 | 102 | Download multiple files without a limit of concurrent connections per host (the default limit is 5): 103 | 104 | \fBwcurl \--curl\-options="\--parallel\-max\-host 0" example.com/filename1.txt example.com/filename2.txt\fP 105 | .SH AUTHORS 106 | .nf 107 | Samuel Henrique \ 108 | Sergio Durigan Junior \ 109 | and many contributors, see the AUTHORS file. 110 | .fi 111 | .SH REPORTING BUGS 112 | If you experience any problems with \fBwcurl\fP that you do not experience with 113 | curl, submit an issue on GitHub: https://github.com/curl/wcurl 114 | .SH COPYRIGHT 115 | \fBwcurl\fP is licensed under the curl license 116 | .SH SEE ALSO 117 | .BR curl (1), 118 | .BR trurl (1) 119 | -------------------------------------------------------------------------------- /wcurl.md: -------------------------------------------------------------------------------- 1 | --- 2 | c: Copyright (C) Samuel Henrique , Sergio Durigan Junior and many contributors, see the AUTHORS file. 3 | SPDX-License-Identifier: curl 4 | Title: wcurl 5 | Section: 1 6 | Source: wcurl 7 | See-also: 8 | - curl (1) 9 | - trurl (1) 10 | Added-in: n/a 11 | --- 12 | 13 | # NAME 14 | 15 | **wcurl** - a simple wrapper around curl to easily download files. 16 | 17 | # SYNOPSIS 18 | 19 | **wcurl \...** 20 | 21 | **wcurl [--curl-options \]... [--dry-run] [--no-decode-filename] [-o|-O|--output \] [--] \...** 22 | 23 | **wcurl [--curl-options=\]... [--dry-run] [--no-decode-filename] [--output=\] [--] \...** 24 | 25 | **wcurl -V|--version** 26 | 27 | **wcurl -h|--help** 28 | 29 | # DESCRIPTION 30 | 31 | **wcurl** is a simple curl wrapper which lets you use curl to download files 32 | without having to remember any parameters. 33 | 34 | Simply call **wcurl** with a list of URLs you want to download and **wcurl** 35 | picks sane defaults. 36 | 37 | If you need anything more complex, you can provide any of curl's supported 38 | parameters via the **--curl-options** option. Just beware that you likely 39 | should be using curl directly if your use case is not covered. 40 | 41 | By default, **wcurl** does: 42 | 43 | ## * Percent-encode whitespace in URLs; 44 | 45 | ## * Download multiple URLs in parallel 46 | if the installed curl's version is \>= 7.66.0 (--parallel); 47 | 48 | ## * Use a total number of 5 parallel connections to the same protocol + hostname + port number target 49 | if the installed curl's version is \>= 8.16.0 (--parallel-max-host); 50 | 51 | ## * Follow redirects; 52 | 53 | ## * Automatically choose a filename as output; 54 | 55 | ## * Avoid overwriting files 56 | if the installed curl's version is \>= 7.83.0 (--no-clobber); 57 | 58 | ## * Perform retries; 59 | 60 | ## * Set the downloaded file timestamp 61 | to the value provided by the server, if available; 62 | 63 | ## * Default to https 64 | if the URL does not contain any scheme; 65 | 66 | ## * Disable curl's URL globbing parser 67 | so {} and [] characters in URLs are not treated specially; 68 | 69 | ## * Percent-decode the resulting filename; 70 | 71 | ## * Use 'index.html' as the default filename 72 | if there is none in the URL. 73 | 74 | # OPTIONS 75 | 76 | ## --curl-options, --curl-options=\... 77 | 78 | Specify extra options to be passed when invoking curl. May be specified more 79 | than once. 80 | 81 | ## -o, -O, --output, --output=\ 82 | 83 | Use the provided output path instead of getting it from the URL. If multiple 84 | URLs are provided, resulting files share the same name with a number appended to 85 | the end (curl \>= 7.83.0). If this option is provided multiple times, only the 86 | last value is considered. 87 | 88 | ## --no-decode-filename 89 | 90 | Do not percent-decode the output filename, even if the percent-encoding in the 91 | URL was done by **wcurl**, e.g.: The URL contained whitespace. 92 | 93 | ## --dry-run 94 | 95 | Do not actually execute curl, just print what would be invoked. 96 | 97 | ## -V, \--version 98 | 99 | Print version information. 100 | 101 | ## -h, \--help 102 | 103 | Print help message. 104 | 105 | # CURL_OPTIONS 106 | 107 | Any option supported by curl can be set here. This is not used by **wcurl**; it 108 | is instead forwarded to the curl invocation. 109 | 110 | # URL 111 | 112 | URL to be downloaded. Anything that is not a parameter is considered 113 | an URL. Whitespace is percent-encoded and the URL is passed to curl, which 114 | then performs the parsing. May be specified more than once. 115 | 116 | # EXAMPLES 117 | 118 | Download a single file: 119 | 120 | **wcurl example.com/filename.txt** 121 | 122 | Download two files in parallel: 123 | 124 | **wcurl example.com/filename1.txt example.com/filename2.txt** 125 | 126 | Download a file passing the **--progress-bar** and **--http2** flags to curl: 127 | 128 | **wcurl --curl-options="--progress-bar --http2" example.com/filename.txt** 129 | 130 | * Resume from an interrupted download. The options necessary to resume the download (`--clobber --continue-at -`) must be the **last** options specified in `--curl-options`. Note that the only way to resume interrupted downloads is to allow wcurl to overwrite the destination file: 131 | 132 | **wcurl --curl-options="--clobber --continue-at -" example.com/filename.txt** 133 | 134 | Download multiple files without a limit of concurrent connections per host (the default limit is 5): 135 | 136 | **wcurl --curl-options="--parallel-max-host 0" example.com/filename1.txt example.com/filename2.txt** 137 | 138 | # AUTHORS 139 | 140 | Samuel Henrique \ 141 | Sergio Durigan Junior \ 142 | and many contributors, see the AUTHORS file. 143 | 144 | # REPORTING BUGS 145 | 146 | If you experience any problems with **wcurl** that you do not experience with 147 | curl, submit an issue on GitHub: https://github.com/curl/wcurl 148 | 149 | # COPYRIGHT 150 | 151 | **wcurl** is licensed under the curl license 152 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | # Changelog 10 | 11 | ## [v2025.11.09] 12 | * Really fix CVE-2025-11563: The patch from v2025.11.04 did not fix the CVE and 13 | the unit test verifying it was broken. 14 | * Replace `>/dev/stderr` with `>&2` for portability. 15 | 16 | ## [v2025.11.04] 17 | * Fix CVE-2025-11563: Do not percent-decode `/` and `\` in output file name to 18 | avoid path traversal. 19 | * Fix typos reported by pyspelling. 20 | * Multiple improvements to GitHub Actions. 21 | 22 | ## [v2025.09.27] 23 | * Set `parallel-max-host` to 5 if curl>=8.16.0. 24 | * Fix example for `--continue-at`. 25 | * Consistent variable names for feature checks. 26 | * Set `CURL_OPTIONS` right before the URL, allowing override of output file name. 27 | * Apply `shfmt` in all shellscript files. 28 | * Minor Markdown tweaks in README.md. 29 | * Update installation instructions. 30 | * Fix typos. 31 | * Update AUTHORS. 32 | 33 | ## [v2025.05.26] 34 | * Increase number of retries to 5 (32 sec total time), fixing the problem with 35 | misleading output. Previously, it was showing a higher number of retries 36 | than what would be done and it always did only 3. 37 | 38 | ## [v2025.04.20] 39 | * Update manpage, help output, README and comments, fixing typos and 40 | standardizing to curl's documentation format. 41 | 42 | ## [v2025.02.24] 43 | * Allow `-o` and `-O` to be used without whitespace (e.g.: `-oNAME`). 44 | * Fix capitalization of the name of copyright owner sergiodj. 45 | * Use the standard copyright header in manpage. 46 | * Create a GitHub workflow for tests and linting. 47 | * Add missing breakline to README to fix formatting. 48 | * Update manpage to describe that `--output` can be used without the equal sign. 49 | * Add installation instructions to README. 50 | * Fix punctuation in the list of features. 51 | * Throw an error message on tests if shunit's version is lower than 2.1.8. 52 | * Update AUTHORS. 53 | 54 | ## [v2024.12.08] 55 | * New parameter `-o|-O|--output|output=` which allows the user to choose the output filename. 56 | * Default to `index.html` as filename if none can be inferred from the URL. 57 | * Percent-decode output filenames by default. 58 | * New option to disable percent-decoding of output filenames: `--no-decode-filename`. 59 | * Fix typo in the list of features of the manpage. 60 | * README/manpage: Point to the curl issue tracker. 61 | * README: 62 | - Add a missing dash to the `--dry-run` command. 63 | - Add a logo. 64 | - Add a brief section explaining about our testsuite. 65 | - Remove HTML `` anchors. 66 | * Symlink LICENSE to LICENSES/curl.txt. 67 | * Update AUTHORS. 68 | 69 | ## [v2024.07.10] 70 | * Change versioning to use dots as separators instead of dashes: 71 | - Previous version: `2024-07-07`. 72 | - New version: `2024.07.10`. 73 | * Support older curl releases, minimum required version is now 7.46.0: 74 | - Only set `--no-clobber` if curl is 7.83 or newer. 75 | - Only set `--parallel` if curl is 7.66 or newer. 76 | * Set `--fail` when invoking curl, in order to display possible errors instead of saving them as 77 | output files. 78 | * Add more tests. 79 | * Remove the need for GNU coreutils' `realpath` for tests. 80 | * Update manpage with links to GitHub and Debian's Salsa. 81 | * Update LICENSE file with new contributors. 82 | 83 | ## [v2024-07-07] 84 | * Drop `getopt` usage, non-GNU/Linux environments are supported now. 85 | * Replace `-o`/`--opts=` parameters with `--curl-options`/`--curl-options=`. 86 | This alternative is more descriptive and it does not coincide with any of curl's parameters. 87 | * Stop auto-resuming downloads and do not overwrite files instead by default. 88 | Safer alternative as otherwise curl can corrupt a file if the name clashes and the size of the existing one is smaller. 89 | One can easily change that behavior with `--curl-options="--continue-at -"`. 90 | * New `--dry-run` option: just print what would be invoked. 91 | * Choose HTTPS as a default protocol, in case there is none in the URL. 92 | * Disable curl's URL globbing parser so `{}` and `[]` characters in URLs are not treated specially. 93 | * Implement support for `--`. 94 | * Implement `-V`/`--version` options. 95 | * Basic testsuite implemented. 96 | * Update manpage, README and help output. 97 | 98 | ## [v2024-07-02] 99 | * First "public" release, announcing the project. 100 | * Use `exec` instead of `eval`. 101 | * Only set `--parallel` if there is more than one URL. 102 | * Fix manpage typo. 103 | * Update COPYRIGHT and AUTHORS in manpage. 104 | * Rewrite wcurl to remove bash dependency, it is now a POSIX shell script. 105 | * Add README.md. 106 | * Add LICENSE. 107 | 108 | ## [v2024-06-26] 109 | * Simplify `--help` output. 110 | * Download multiple URLs in parallel. 111 | * Use remote timestamp for output file. 112 | * Update REPORTING BUGS section of the manpage. 113 | 114 | ## [v2024-05-14] 115 | * First release. 116 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # wcurl - a simple wrapper around curl to easily download files. 2 | # 3 | # This is wcurl's test pipeline. 4 | # 5 | # Copyright (C) Samuel Henrique , Sergio Durigan 6 | # Junior and many contributors, see the AUTHORS 7 | # file. 8 | # 9 | # Permission to use, copy, modify, and distribute this software for any purpose 10 | # with or without fee is hereby granted, provided that the above copyright 11 | # notice and this permission notice appear in all copies. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN 16 | # NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | # OR OTHER DEALINGS IN THE SOFTWARE. 20 | # 21 | # Except as contained in this notice, the name of a copyright holder shall not be 22 | # used in advertising or otherwise to promote the sale, use or other dealings in 23 | # this Software without prior written authorization of the copyright holder. 24 | # 25 | # SPDX-License-Identifier: curl 26 | --- 27 | name: Test 28 | 29 | on: 30 | push: 31 | branches: 32 | - main 33 | pull_request: 34 | branches: 35 | - main 36 | 37 | concurrency: 38 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} 39 | cancel-in-progress: true 40 | 41 | permissions: {} 42 | 43 | jobs: 44 | lint: 45 | name: lint 46 | runs-on: ubuntu-latest 47 | steps: 48 | - name: Checkout repository 49 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 50 | with: 51 | persist-credentials: false 52 | 53 | - name: Install tools 54 | run: | 55 | /home/linuxbrew/.linuxbrew/bin/brew install zizmor shellcheck checkbashisms shfmt typos-cli 56 | python3 -m venv ~/venv 57 | ~/venv/bin/pip --disable-pip-version-check --no-input --no-cache-dir install --progress-bar off --prefer-binary \ 58 | -r .github/workflows/requirements.txt 59 | 60 | - name: 'REUSE check' 61 | run: | 62 | source ~/venv/bin/activate 63 | reuse lint 64 | 65 | - name: Run zizmor 66 | env: 67 | GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' 68 | run: | 69 | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" 70 | zizmor --pedantic .github/workflows/*.yml .github/dependabot.yml 71 | 72 | - name: Run shellcheck 73 | run: | 74 | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" 75 | shellcheck --version 76 | git grep -z -l '^#!/bin/sh' | xargs -0 -r shellcheck -- 77 | 78 | - name: Run checkbashisms 79 | run: | 80 | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" 81 | checkbashisms --version 82 | git grep -z -l '^#!/bin/sh' | xargs -0 -r checkbashisms -- 83 | 84 | - name: Run shfmt 85 | run: | 86 | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" 87 | shfmt --version 88 | shfmt --func-next-line --space-redirects --case-indent --binary-next-line --indent 4 --posix --diff . 89 | 90 | - name: Run codespell 91 | run: | 92 | source ~/venv/bin/activate 93 | codespell --version 94 | codespell 95 | 96 | - name: Run typos 97 | run: | 98 | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" 99 | typos --version 100 | typos 101 | 102 | debian: 103 | name: debian 104 | runs-on: ubuntu-latest 105 | container: 106 | image: debian:stable # zizmor: ignore[unpinned-images] 107 | steps: 108 | - name: Install curl and shunit2 109 | run: | 110 | apt-get -o Dpkg::Use-Pty=0 update 111 | apt-get -o Dpkg::Use-Pty=0 install -y --no-install-suggests --no-install-recommends curl shunit2 112 | 113 | - name: Checkout repository 114 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 115 | with: 116 | persist-credentials: false 117 | 118 | - name: Run shunit2 tests 119 | run: ./tests/tests.sh 120 | 121 | fedora: 122 | name: fedora 123 | runs-on: ubuntu-latest 124 | container: 125 | image: fedora:latest # zizmor: ignore[unpinned-images] 126 | steps: 127 | - name: Install git and shunit2 128 | run: | 129 | dnf install -y git 130 | git clone \ 131 | --depth 1 --branch v2.1.8 \ 132 | https://github.com/kward/shunit2.git 133 | cd shunit2 134 | cp shunit2 /usr/local/bin/shunit2 135 | 136 | - name: Checkout repository 137 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 138 | with: 139 | persist-credentials: false 140 | 141 | - name: Run shunit2 tests 142 | run: ./tests/tests.sh 143 | 144 | macos: 145 | name: macos 146 | runs-on: macos-latest 147 | steps: 148 | - name: Install shunit2 149 | run: HOMEBREW_NO_AUTO_UPDATE=1 brew install shunit2 150 | 151 | - name: Checkout repository 152 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 153 | with: 154 | persist-credentials: false 155 | 156 | - name: Run shunit2 tests 157 | run: ./tests/tests.sh 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | # [![wcurl logo](https://curl.se/logo/wcurl-logo.svg)](https://curl.se/wcurl) 10 | 11 | # Install wcurl 12 | 13 | First check if your distro/OS vendor ships `wcurl` as part of their official 14 | repositories, `wcurl` might be shipped as part of the `curl` package. 15 | 16 | If they do not ship it, consider making a request for it. 17 | 18 | You can always install wcurl by simply downloading the script: 19 | 20 | ```sh 21 | curl -fLO https://github.com/curl/wcurl/releases/latest/download/wcurl 22 | chmod +x wcurl 23 | sudo mv wcurl /usr/local/bin/wcurl 24 | ``` 25 | 26 | # Install wcurl's manpage 27 | 28 | ```sh 29 | curl -fLO https://github.com/curl/wcurl/releases/latest/download/wcurl.1 30 | sudo mkdir -p /usr/local/share/man/man1/ 31 | sudo mv wcurl.1 /usr/local/share/man/man1/wcurl.1 32 | sudo mandb 33 | ``` 34 | 35 | # wcurl(1) 36 | 37 | **wcurl** 38 | 39 | * a simple wrapper around curl to easily download files. 40 | 41 | # Synopsis 42 | 43 | ```text 44 | wcurl ... 45 | wcurl [--curl-options ]... [--no-decode-filename] [-o|-O|--output ] [--dry-run] [--] ... 46 | wcurl [--curl-options=]... [--no-decode-filename] [--output=] [--dry-run] [--] ... 47 | wcurl -V|--version 48 | wcurl -h|--help 49 | ``` 50 | 51 | # Description 52 | 53 | **wcurl** is a simple curl wrapper which lets you use curl to download files 54 | without having to remember any parameters. 55 | 56 | Simply call **wcurl** with a list of URLs you want to download and **wcurl** picks 57 | sane defaults. 58 | 59 | If you need anything more complex, you can provide any of curl's supported 60 | parameters via the `--curl-options` option. Just beware that you likely 61 | should be using curl directly if your use case is not covered. 62 | 63 | * By default, **wcurl** does: 64 | * Percent-encode whitespace in URLs; 65 | * Download multiple URLs in parallel if the installed curl's version is >= 7.66.0 (`--parallel`); 66 | * Use a total number of 5 parallel connections to the same protocol + hostname + port number target if the installed curl's version is >= 8.16.0 (`--parallel-max-host`); 67 | * Follow redirects; 68 | * Automatically choose a filename as output; 69 | * Avoid overwriting files if the installed curl's version is >= 7.83.0 (`--no-clobber`); 70 | * Perform retries; 71 | * Set the downloaded file timestamp to the value provided by the server, if available; 72 | * Disable **curl**'s URL globbing parser so `{}` and `[]` characters in URLs are not treated specially; 73 | * Percent-decode the resulting filename; 74 | * Use "index.html" as default filename if there is none in the URL. 75 | 76 | # Options 77 | 78 | * `--curl-options, curl-options=`... 79 | 80 | Specify extra options to be passed when invoking curl. May be specified more than once. 81 | 82 | * `-o, -O, --output, --output=` 83 | 84 | Use the provided output path instead of getting it from the URL. If multiple 85 | URLs are provided, resulting files share the same name with a number appended to 86 | the end (curl >= 7.83.0). If this option is provided multiple times, only the 87 | last value is considered. 88 | 89 | * `--no-decode-filename` 90 | 91 | Do not percent-decode the output filename, even if the percent-encoding in the 92 | URL was done by wcurl, e.g.: The URL contained whitespace. 93 | 94 | * `--dry-run` 95 | 96 | Do not actually execute curl, just print what would be invoked. 97 | 98 | * `-V, --version` 99 | 100 | Print version information. 101 | 102 | * `-h, --help` 103 | 104 | Print help message. 105 | 106 | # CURL_OPTIONS 107 | 108 | Any option supported by curl can be set here. This is not used by wcurl; it is 109 | instead forwarded to the curl invocation. 110 | 111 | # URL 112 | 113 | URL to be downloaded. Anything that is not a parameter is considered 114 | an URL. Whitespace is percent-encoded and the URL is passed to curl, which 115 | then performs the parsing. May be specified more than once. 116 | 117 | # Examples 118 | 119 | * Download a single file: 120 | 121 | ```sh 122 | wcurl example.com/filename.txt 123 | ``` 124 | 125 | * Download two files in parallel: 126 | 127 | ```sh 128 | wcurl example.com/filename1.txt example.com/filename2.txt 129 | ``` 130 | 131 | * Download a file passing the `--progress-bar` and `--http2` flags to curl: 132 | 133 | ```sh 134 | wcurl --curl-options="--progress-bar --http2" example.com/filename.txt 135 | ``` 136 | 137 | * Resume from an interrupted download. The options necessary to resume the download (`--clobber --continue-at -`) must be the **last** options specified in `--curl-options`. Note that the only way to resume interrupted downloads is to allow wcurl to overwrite the destination file: 138 | 139 | ```sh 140 | wcurl --curl-options="--clobber --continue-at -" example.com/filename.txt 141 | ``` 142 | 143 | * Download multiple files without a limit of concurrent connections per host (the default limit is 5): 144 | 145 | ```sh 146 | wcurl --curl-options="--parallel-max-host 0" example.com/filename1.txt example.com/filename2.txt 147 | ``` 148 | 149 | # Running the testsuite 150 | 151 | If you would like to run the tests, you first need to install the 152 | `shunit2` package. On Debian-like and Fedora-like systems, the 153 | package is called `shunit2`. 154 | 155 | After that, you can run the testsuite by simply invoking the test 156 | script: 157 | 158 | ```sh 159 | ./tests/tests.sh 160 | ``` 161 | 162 | # Lint 163 | 164 | To lint the shell scripts, you need to install `shellcheck` and `checkbashisms`. These tools check the scripts for issues and ensure they follow best practices. 165 | 166 | * On Debian-like systems: `apt install shellcheck devscripts` 167 | * On Fedora-like systems: `dnf install shellcheck devscripts` 168 | 169 | After installation, you can run `shellcheck` and `checkbashisms` by executing the following commands: 170 | 171 | ```sh 172 | shellcheck wcurl ./tests/* 173 | 174 | checkbashisms wcurl ./tests/* 175 | ``` 176 | 177 | # Authors 178 | 179 | Samuel Henrique <[samueloph@debian.org](mailto:samueloph@debian.org)> 180 | Sergio Durigan Junior <[sergiodj@debian.org](mailto:sergiodj@debian.org)> 181 | and many contributors, see the AUTHORS file. 182 | 183 | # Reporting Bugs 184 | 185 | If you experience any problems with **wcurl** that you do not experience with curl, 186 | submit an issue [here](https://github.com/curl/wcurl/issues). 187 | 188 | # Copyright 189 | 190 | **wcurl** is licensed under the curl license 191 | -------------------------------------------------------------------------------- /tests/tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # wcurl - a simple wrapper around curl to easily download files. 4 | # 5 | # This is wcurl's testsuite. 6 | # 7 | # Copyright (C) Samuel Henrique , Sergio Durigan 8 | # Junior and many contributors, see the AUTHORS 9 | # file. 10 | # 11 | # Permission to use, copy, modify, and distribute this software for any purpose 12 | # with or without fee is hereby granted, provided that the above copyright 13 | # notice and this permission notice appear in all copies. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN 18 | # NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 19 | # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 | # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 21 | # OR OTHER DEALINGS IN THE SOFTWARE. 22 | # 23 | # Except as contained in this notice, the name of a copyright holder shall not be 24 | # used in advertising or otherwise to promote the sale, use or other dealings in 25 | # this Software without prior written authorization of the copyright holder. 26 | # 27 | # SPDX-License-Identifier: curl 28 | 29 | ROOTDIR=$(CDPATH="" cd -- "$(dirname -- "$0")/.." && pwd) 30 | readonly ROOTDIR 31 | export PATH="${ROOTDIR}:${PATH}" 32 | 33 | readonly WCURL_CMD="wcurl --dry-run " 34 | 35 | oneTimeSetUp() 36 | { 37 | if ! assertContains "Check compatibility" "test" "test"; then 38 | echo "Error: shunit2 version 2.1.8 or higher is required." 39 | echo "Please install a compatible version of shunit2." 40 | exit 1 41 | fi 42 | } 43 | 44 | debug() 45 | { 46 | if [ -n "${DEBUG}" ]; then 47 | printf "D: %s\n" "$*" 48 | fi 49 | } 50 | 51 | testUsage() 52 | { 53 | ret=$(${WCURL_CMD} --help) 54 | assertTrue "Verify whether '--help' option exits successfully" "$?" 55 | 56 | debug "Verifying: '${ret}'" 57 | printf "%s\n" "${ret}" | grep -qF "wcurl -- a simple wrapper around curl to easily download files" 58 | assertTrue "Verify whether the usage command works" "$?" 59 | } 60 | 61 | testNoOptionError() 62 | { 63 | ret=$(${WCURL_CMD} 2>&1) 64 | assertFalse "Verify whether 'wcurl' without options exits with an error" "$?" 65 | assertEquals "Verify whether 'wcurl' without options displays an error message" "${ret}" "You must provide at least one URL to download." 66 | } 67 | 68 | testInvalidOptionError() 69 | { 70 | invalidoption="--frobnicator" 71 | ret=$(${WCURL_CMD} ${invalidoption} 2>&1) 72 | assertFalse "Verify whether 'wcurl' with an invalid option exits with an error" "$?" 73 | assertEquals "Verify whether 'wcurl' with an invalid option displays an error message" "${ret}" "Unknown option: '${invalidoption}'." 74 | } 75 | 76 | testParallelIfMoreThanOneUrl() 77 | { 78 | # TODO: This test is wrong for curl 7.65 or older, since --parallel was only introduced in 7.66. 79 | # We should check curl's version and skip this test instead. 80 | url_1='example.com/1' 81 | url_2='example.com/2' 82 | ret=$(${WCURL_CMD} ${url_1} ${url_2}) 83 | assertContains "Verify whether 'wcurl' uses '--parallel' if more than one url is provided" "${ret}" '--parallel' 84 | } 85 | 86 | testEncodingWhitespace() 87 | { 88 | url='example.com/white space' 89 | ret=$(${WCURL_CMD} "${url}") 90 | assertContains "Verify 'wcurl' encodes spaces in URLs as '%20'" "${ret}" 'example.com/white%20space' 91 | } 92 | 93 | testDoubleDash() 94 | { 95 | params='example.com --curl-options=abc' 96 | ret=$(${WCURL_CMD} -- "${params}") 97 | assertTrue "Verify whether 'wcurl' accepts '--' without erroring" "$?" 98 | assertContains "Verify whether 'wcurl' considers everywhing after '--' a url" "${ret}" '--curl-options=abc' 99 | } 100 | 101 | testCurlOptions() 102 | { 103 | params='example.com --curl-options=--foo --curl-options --bar' 104 | ret=$(${WCURL_CMD} "${params}") 105 | assertTrue "Verify 'wcurl' accepts '--curl-options' with and without trailing '='" "$?" 106 | assertContains "Verify 'wcurl' correctly passes through --curl-options=