├── .github └── workflows │ ├── ci.yaml │ └── release.yaml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE.txt ├── README.md └── src ├── lib.rs ├── main.rs ├── redirect.rs ├── replace.rs ├── resolve.rs ├── test_helper.rs └── url.rs /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | tests: 6 | name: Unit tests 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest, macos-latest, windows-latest] 10 | fail-fast: false 11 | runs-on: ${{ matrix.os }} 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions-rs/toolchain@v1 15 | with: 16 | profile: minimal 17 | toolchain: stable 18 | override: true 19 | - name: Run unit tests 20 | uses: actions-rs/cargo@v1 21 | with: 22 | command: test 23 | args: --color always 24 | - run: cargo run -- --help 25 | lint: 26 | name: Rustfmt and Clippy 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@v2 30 | - uses: actions-rs/toolchain@v1 31 | with: 32 | profile: minimal 33 | toolchain: stable 34 | override: true 35 | components: clippy, rustfmt 36 | - name: Clippy 37 | uses: actions-rs/cargo@v1 38 | with: 39 | command: clippy 40 | args: --color always --tests -- -D warnings 41 | - run: rustup component add rustfmt 42 | - name: rustfmt 43 | uses: actions-rs/cargo@v1 44 | with: 45 | command: fmt 46 | args: -- --check --color always 47 | docker: 48 | name: Dockerfile 49 | runs-on: ubuntu-latest 50 | steps: 51 | - uses: actions/checkout@v2 52 | - name: Build image 53 | id: image 54 | uses: docker/build-push-action@v2 55 | with: 56 | push: false 57 | - name: Test Docker image 58 | run: | 59 | want="https://github.com/rhysd/vim-crystal" 60 | have="$(echo "https://github.com/rhysd/vim-crystal" | docker run -i --rm ${{ steps.image.outputs.digest }})" 61 | if [[ "$have" != "$want" ]]; then 62 | echo "expected ${want} but got ${have}" >&2 63 | exit 1 64 | fi 65 | - name: Lint Dockerfile with hadolint 66 | run: docker run --rm -i hadolint/hadolint hadolint --ignore DL3007 --ignore DL3008 --strict-labels - < Dockerfile 67 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | 7 | jobs: 8 | docker: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Get tag name 13 | id: tag 14 | run: | 15 | echo "::set-output name=name::${GITHUB_REF#refs/tags/v}" 16 | - name: Login to DockerHub 17 | uses: docker/login-action@v1 18 | with: 19 | username: rhysd 20 | password: ${{ secrets.DOCKERHUB_TOKEN }} 21 | - name: Build and push 22 | uses: docker/build-push-action@v2 23 | with: 24 | push: true 25 | tags: | 26 | rhysd/fixred:${{ steps.tag.outputs.name }} 27 | rhysd/fixred:latest 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | # [v1.1.4](https://github.com/rhysd/fixred/releases/tag/v1.1.4) - 21 Oct 2021 3 | 4 | - Fix `cargo install fixred` failed due to new release of `clap` crate v3.0.0-beta.5 5 | 6 | [Changes][v1.1.4] 7 | 8 | 9 | 10 | # [v1.1.3](https://github.com/rhysd/fixred/releases/tag/v1.1.3) - 30 Sep 2021 11 | 12 | - Ignore non-UTF8 files. This is useful when you specify some directories to update files recursively. Now fixred fixes only UTF-8 files in directories recursively. Previously fixred stopped when finding a non-UTF8 file. To know which files were ignored, try `--verbose` flag. 13 | 14 | [Changes][v1.1.3] 15 | 16 | 17 | 18 | # [v1.1.2](https://github.com/rhysd/fixred/releases/tag/v1.1.2) - 29 Sep 2021 19 | 20 | - Dependencies necessary only for building `fixred` executable are now optional. By removing `executable` feature, they can be omit on installing this tool as Rust library for less dependencies. 21 | ```toml 22 | [dependencies] 23 | fixred = { version = "1", default-features = false, features = [] } 24 | ``` 25 | - Add an introduction dedicated for [the API document](https://docs.rs/fixred/). Previously `README.md` file at root of this repository was used but it is basically for `fixred` executable. 26 | 27 | [Changes][v1.1.2] 28 | 29 | 30 | 31 | # [v1.1.1](https://github.com/rhysd/fixred/releases/tag/v1.1.1) - 29 Sep 2021 32 | 33 | - Now fixred is available as Rust library. See [the API document](https://docs.rs/crate/fixred) 34 | 35 | [Changes][v1.1.1] 36 | 37 | 38 | 39 | # [v1.1.0](https://github.com/rhysd/fixred/releases/tag/v1.1.0) - 28 Sep 2021 40 | 41 | - Release [the docker image](https://hub.docker.com/r/rhysd/fixred). 42 | - Use a dedicated `$FIXRED_LOG` environment variable instead of `$RUST_LOG` environment variable to control logs. 43 | - Add `--verbose` flag to show info level logs easily. 44 | 45 | [Changes][v1.1.0] 46 | 47 | 48 | 49 | # [v1.0.2](https://github.com/rhysd/fixred/releases/tag/v1.0.2) - 28 Sep 2021 50 | 51 | - Fix percent-encoded characters are ignored 52 | - Reduce number of dependencies from 86 to 77 53 | 54 | [Changes][v1.0.2] 55 | 56 | 57 | 58 | # [v1.0.1](https://github.com/rhysd/fixred/releases/tag/v1.0.1) - 27 Sep 2021 59 | 60 | - Keep fragments in URLs 61 | 62 | [Changes][v1.0.1] 63 | 64 | 65 | 66 | # [v1.0.0](https://github.com/rhysd/fixred/releases/tag/v1.0.0) - 27 Sep 2021 67 | 68 | First release :tada: 69 | 70 | See the document to know how to install fixred: https://github.com/rhysd/fixred#installation 71 | 72 | [Changes][v1.0.0] 73 | 74 | 75 | [v1.1.4]: https://github.com/rhysd/fixred/compare/v1.1.3...v1.1.4 76 | [v1.1.3]: https://github.com/rhysd/fixred/compare/v1.1.2...v1.1.3 77 | [v1.1.2]: https://github.com/rhysd/fixred/compare/v1.1.1...v1.1.2 78 | [v1.1.1]: https://github.com/rhysd/fixred/compare/v1.1.0...v1.1.1 79 | [v1.1.0]: https://github.com/rhysd/fixred/compare/v1.0.2...v1.1.0 80 | [v1.0.2]: https://github.com/rhysd/fixred/compare/v1.0.1...v1.0.2 81 | [v1.0.1]: https://github.com/rhysd/fixred/compare/v1.0.0...v1.0.1 82 | [v1.0.0]: https://github.com/rhysd/fixred/tree/v1.0.0 83 | 84 | 85 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "0.7.18" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anyhow" 16 | version = "1.0.44" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" 19 | 20 | [[package]] 21 | name = "atty" 22 | version = "0.2.14" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 25 | dependencies = [ 26 | "hermit-abi", 27 | "libc", 28 | "winapi", 29 | ] 30 | 31 | [[package]] 32 | name = "autocfg" 33 | version = "1.0.1" 34 | source = "registry+https://github.com/rust-lang/crates.io-index" 35 | checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" 36 | 37 | [[package]] 38 | name = "bitflags" 39 | version = "1.3.2" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 42 | 43 | [[package]] 44 | name = "cc" 45 | version = "1.0.71" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" 48 | 49 | [[package]] 50 | name = "cfg-if" 51 | version = "1.0.0" 52 | source = "registry+https://github.com/rust-lang/crates.io-index" 53 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 54 | 55 | [[package]] 56 | name = "chashmap" 57 | version = "2.2.2" 58 | source = "registry+https://github.com/rust-lang/crates.io-index" 59 | checksum = "ff41a3c2c1e39921b9003de14bf0439c7b63a9039637c291e1a64925d8ddfa45" 60 | dependencies = [ 61 | "owning_ref", 62 | "parking_lot", 63 | ] 64 | 65 | [[package]] 66 | name = "clap" 67 | version = "3.0.0-beta.5" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "feff3878564edb93745d58cf63e17b63f24142506e7a20c87a5521ed7bfb1d63" 70 | dependencies = [ 71 | "atty", 72 | "bitflags", 73 | "indexmap", 74 | "os_str_bytes", 75 | "strsim", 76 | "termcolor", 77 | "textwrap", 78 | ] 79 | 80 | [[package]] 81 | name = "crossbeam-channel" 82 | version = "0.5.1" 83 | source = "registry+https://github.com/rust-lang/crates.io-index" 84 | checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" 85 | dependencies = [ 86 | "cfg-if", 87 | "crossbeam-utils", 88 | ] 89 | 90 | [[package]] 91 | name = "crossbeam-deque" 92 | version = "0.8.1" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" 95 | dependencies = [ 96 | "cfg-if", 97 | "crossbeam-epoch", 98 | "crossbeam-utils", 99 | ] 100 | 101 | [[package]] 102 | name = "crossbeam-epoch" 103 | version = "0.9.5" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" 106 | dependencies = [ 107 | "cfg-if", 108 | "crossbeam-utils", 109 | "lazy_static", 110 | "memoffset", 111 | "scopeguard", 112 | ] 113 | 114 | [[package]] 115 | name = "crossbeam-utils" 116 | version = "0.8.8" 117 | source = "registry+https://github.com/rust-lang/crates.io-index" 118 | checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" 119 | dependencies = [ 120 | "cfg-if", 121 | "lazy_static", 122 | ] 123 | 124 | [[package]] 125 | name = "curl" 126 | version = "0.4.39" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "aaa3b8db7f3341ddef15786d250106334d4a6c4b0ae4a46cd77082777d9849b9" 129 | dependencies = [ 130 | "curl-sys", 131 | "libc", 132 | "openssl-probe", 133 | "openssl-sys", 134 | "schannel", 135 | "socket2", 136 | "winapi", 137 | ] 138 | 139 | [[package]] 140 | name = "curl-sys" 141 | version = "0.4.49+curl-7.79.1" 142 | source = "registry+https://github.com/rust-lang/crates.io-index" 143 | checksum = "e0f44960aea24a786a46907b8824ebc0e66ca06bf4e4978408c7499620343483" 144 | dependencies = [ 145 | "cc", 146 | "libc", 147 | "libz-sys", 148 | "openssl-sys", 149 | "pkg-config", 150 | "vcpkg", 151 | "winapi", 152 | ] 153 | 154 | [[package]] 155 | name = "either" 156 | version = "1.6.1" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" 159 | 160 | [[package]] 161 | name = "env_logger" 162 | version = "0.9.0" 163 | source = "registry+https://github.com/rust-lang/crates.io-index" 164 | checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" 165 | dependencies = [ 166 | "atty", 167 | "humantime", 168 | "log", 169 | "termcolor", 170 | ] 171 | 172 | [[package]] 173 | name = "fixred" 174 | version = "1.1.4" 175 | dependencies = [ 176 | "aho-corasick", 177 | "anyhow", 178 | "chashmap", 179 | "clap", 180 | "curl", 181 | "env_logger", 182 | "log", 183 | "rayon", 184 | "regex", 185 | "walkdir", 186 | ] 187 | 188 | [[package]] 189 | name = "fuchsia-cprng" 190 | version = "0.1.1" 191 | source = "registry+https://github.com/rust-lang/crates.io-index" 192 | checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 193 | 194 | [[package]] 195 | name = "hashbrown" 196 | version = "0.11.2" 197 | source = "registry+https://github.com/rust-lang/crates.io-index" 198 | checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" 199 | 200 | [[package]] 201 | name = "hermit-abi" 202 | version = "0.1.19" 203 | source = "registry+https://github.com/rust-lang/crates.io-index" 204 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 205 | dependencies = [ 206 | "libc", 207 | ] 208 | 209 | [[package]] 210 | name = "humantime" 211 | version = "2.1.0" 212 | source = "registry+https://github.com/rust-lang/crates.io-index" 213 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 214 | 215 | [[package]] 216 | name = "indexmap" 217 | version = "1.7.0" 218 | source = "registry+https://github.com/rust-lang/crates.io-index" 219 | checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" 220 | dependencies = [ 221 | "autocfg", 222 | "hashbrown", 223 | ] 224 | 225 | [[package]] 226 | name = "lazy_static" 227 | version = "1.4.0" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 230 | 231 | [[package]] 232 | name = "libc" 233 | version = "0.2.104" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce" 236 | 237 | [[package]] 238 | name = "libz-sys" 239 | version = "1.1.3" 240 | source = "registry+https://github.com/rust-lang/crates.io-index" 241 | checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" 242 | dependencies = [ 243 | "cc", 244 | "libc", 245 | "pkg-config", 246 | "vcpkg", 247 | ] 248 | 249 | [[package]] 250 | name = "log" 251 | version = "0.4.14" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" 254 | dependencies = [ 255 | "cfg-if", 256 | ] 257 | 258 | [[package]] 259 | name = "maybe-uninit" 260 | version = "2.0.0" 261 | source = "registry+https://github.com/rust-lang/crates.io-index" 262 | checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" 263 | 264 | [[package]] 265 | name = "memchr" 266 | version = "2.4.1" 267 | source = "registry+https://github.com/rust-lang/crates.io-index" 268 | checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" 269 | 270 | [[package]] 271 | name = "memoffset" 272 | version = "0.6.4" 273 | source = "registry+https://github.com/rust-lang/crates.io-index" 274 | checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" 275 | dependencies = [ 276 | "autocfg", 277 | ] 278 | 279 | [[package]] 280 | name = "num_cpus" 281 | version = "1.13.0" 282 | source = "registry+https://github.com/rust-lang/crates.io-index" 283 | checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" 284 | dependencies = [ 285 | "hermit-abi", 286 | "libc", 287 | ] 288 | 289 | [[package]] 290 | name = "openssl-probe" 291 | version = "0.1.4" 292 | source = "registry+https://github.com/rust-lang/crates.io-index" 293 | checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" 294 | 295 | [[package]] 296 | name = "openssl-sys" 297 | version = "0.9.67" 298 | source = "registry+https://github.com/rust-lang/crates.io-index" 299 | checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" 300 | dependencies = [ 301 | "autocfg", 302 | "cc", 303 | "libc", 304 | "pkg-config", 305 | "vcpkg", 306 | ] 307 | 308 | [[package]] 309 | name = "os_str_bytes" 310 | version = "4.2.0" 311 | source = "registry+https://github.com/rust-lang/crates.io-index" 312 | checksum = "addaa943333a514159c80c97ff4a93306530d965d27e139188283cd13e06a799" 313 | dependencies = [ 314 | "memchr", 315 | ] 316 | 317 | [[package]] 318 | name = "owning_ref" 319 | version = "0.3.3" 320 | source = "registry+https://github.com/rust-lang/crates.io-index" 321 | checksum = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" 322 | dependencies = [ 323 | "stable_deref_trait", 324 | ] 325 | 326 | [[package]] 327 | name = "parking_lot" 328 | version = "0.4.8" 329 | source = "registry+https://github.com/rust-lang/crates.io-index" 330 | checksum = "149d8f5b97f3c1133e3cfcd8886449959e856b557ff281e292b733d7c69e005e" 331 | dependencies = [ 332 | "owning_ref", 333 | "parking_lot_core", 334 | ] 335 | 336 | [[package]] 337 | name = "parking_lot_core" 338 | version = "0.2.14" 339 | source = "registry+https://github.com/rust-lang/crates.io-index" 340 | checksum = "4db1a8ccf734a7bce794cc19b3df06ed87ab2f3907036b693c68f56b4d4537fa" 341 | dependencies = [ 342 | "libc", 343 | "rand", 344 | "smallvec", 345 | "winapi", 346 | ] 347 | 348 | [[package]] 349 | name = "pkg-config" 350 | version = "0.3.20" 351 | source = "registry+https://github.com/rust-lang/crates.io-index" 352 | checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" 353 | 354 | [[package]] 355 | name = "rand" 356 | version = "0.4.6" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 359 | dependencies = [ 360 | "fuchsia-cprng", 361 | "libc", 362 | "rand_core 0.3.1", 363 | "rdrand", 364 | "winapi", 365 | ] 366 | 367 | [[package]] 368 | name = "rand_core" 369 | version = "0.3.1" 370 | source = "registry+https://github.com/rust-lang/crates.io-index" 371 | checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 372 | dependencies = [ 373 | "rand_core 0.4.2", 374 | ] 375 | 376 | [[package]] 377 | name = "rand_core" 378 | version = "0.4.2" 379 | source = "registry+https://github.com/rust-lang/crates.io-index" 380 | checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 381 | 382 | [[package]] 383 | name = "rayon" 384 | version = "1.5.1" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" 387 | dependencies = [ 388 | "autocfg", 389 | "crossbeam-deque", 390 | "either", 391 | "rayon-core", 392 | ] 393 | 394 | [[package]] 395 | name = "rayon-core" 396 | version = "1.9.1" 397 | source = "registry+https://github.com/rust-lang/crates.io-index" 398 | checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" 399 | dependencies = [ 400 | "crossbeam-channel", 401 | "crossbeam-deque", 402 | "crossbeam-utils", 403 | "lazy_static", 404 | "num_cpus", 405 | ] 406 | 407 | [[package]] 408 | name = "rdrand" 409 | version = "0.4.0" 410 | source = "registry+https://github.com/rust-lang/crates.io-index" 411 | checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 412 | dependencies = [ 413 | "rand_core 0.3.1", 414 | ] 415 | 416 | [[package]] 417 | name = "regex" 418 | version = "1.5.5" 419 | source = "registry+https://github.com/rust-lang/crates.io-index" 420 | checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" 421 | dependencies = [ 422 | "aho-corasick", 423 | "memchr", 424 | "regex-syntax", 425 | ] 426 | 427 | [[package]] 428 | name = "regex-syntax" 429 | version = "0.6.25" 430 | source = "registry+https://github.com/rust-lang/crates.io-index" 431 | checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" 432 | 433 | [[package]] 434 | name = "same-file" 435 | version = "1.0.6" 436 | source = "registry+https://github.com/rust-lang/crates.io-index" 437 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 438 | dependencies = [ 439 | "winapi-util", 440 | ] 441 | 442 | [[package]] 443 | name = "schannel" 444 | version = "0.1.19" 445 | source = "registry+https://github.com/rust-lang/crates.io-index" 446 | checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" 447 | dependencies = [ 448 | "lazy_static", 449 | "winapi", 450 | ] 451 | 452 | [[package]] 453 | name = "scopeguard" 454 | version = "1.1.0" 455 | source = "registry+https://github.com/rust-lang/crates.io-index" 456 | checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" 457 | 458 | [[package]] 459 | name = "smallvec" 460 | version = "0.6.14" 461 | source = "registry+https://github.com/rust-lang/crates.io-index" 462 | checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" 463 | dependencies = [ 464 | "maybe-uninit", 465 | ] 466 | 467 | [[package]] 468 | name = "socket2" 469 | version = "0.4.2" 470 | source = "registry+https://github.com/rust-lang/crates.io-index" 471 | checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" 472 | dependencies = [ 473 | "libc", 474 | "winapi", 475 | ] 476 | 477 | [[package]] 478 | name = "stable_deref_trait" 479 | version = "1.2.0" 480 | source = "registry+https://github.com/rust-lang/crates.io-index" 481 | checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" 482 | 483 | [[package]] 484 | name = "strsim" 485 | version = "0.10.0" 486 | source = "registry+https://github.com/rust-lang/crates.io-index" 487 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 488 | 489 | [[package]] 490 | name = "termcolor" 491 | version = "1.1.2" 492 | source = "registry+https://github.com/rust-lang/crates.io-index" 493 | checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" 494 | dependencies = [ 495 | "winapi-util", 496 | ] 497 | 498 | [[package]] 499 | name = "textwrap" 500 | version = "0.14.2" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" 503 | 504 | [[package]] 505 | name = "vcpkg" 506 | version = "0.2.15" 507 | source = "registry+https://github.com/rust-lang/crates.io-index" 508 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 509 | 510 | [[package]] 511 | name = "walkdir" 512 | version = "2.3.2" 513 | source = "registry+https://github.com/rust-lang/crates.io-index" 514 | checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" 515 | dependencies = [ 516 | "same-file", 517 | "winapi", 518 | "winapi-util", 519 | ] 520 | 521 | [[package]] 522 | name = "winapi" 523 | version = "0.3.9" 524 | source = "registry+https://github.com/rust-lang/crates.io-index" 525 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 526 | dependencies = [ 527 | "winapi-i686-pc-windows-gnu", 528 | "winapi-x86_64-pc-windows-gnu", 529 | ] 530 | 531 | [[package]] 532 | name = "winapi-i686-pc-windows-gnu" 533 | version = "0.4.0" 534 | source = "registry+https://github.com/rust-lang/crates.io-index" 535 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 536 | 537 | [[package]] 538 | name = "winapi-util" 539 | version = "0.1.5" 540 | source = "registry+https://github.com/rust-lang/crates.io-index" 541 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 542 | dependencies = [ 543 | "winapi", 544 | ] 545 | 546 | [[package]] 547 | name = "winapi-x86_64-pc-windows-gnu" 548 | version = "0.4.0" 549 | source = "registry+https://github.com/rust-lang/crates.io-index" 550 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 551 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fixred" 3 | version = "1.1.4" 4 | edition = "2018" 5 | authors = ["rhysd "] 6 | description = "Command line tool to fix outdated URLs in files with redirected ones" 7 | license = "MIT" 8 | homepage = "https://github.com/rhysd/fixred#readme" 9 | repository = "https://github.com/rhysd/fixred" 10 | readme = "README.md" 11 | include = [ 12 | "/src", 13 | "/LICENSE.txt", 14 | "/README.md", 15 | ] 16 | categories = ["command-line-utilities"] 17 | keywords = ["tool", "fixer", "outdated-links"] 18 | 19 | [badges] 20 | maintenance = { status = "passively-maintained" } 21 | 22 | [[bin]] 23 | name = "fixred" 24 | path = "src/main.rs" 25 | required-features = ["executable"] 26 | 27 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 28 | 29 | [dependencies] 30 | aho-corasick = "0.7" 31 | anyhow = "1" 32 | chashmap = "2" 33 | clap = { version = "3.0.0-beta", default-features = false, features = ["std", "color", "suggestions"], optional = true } 34 | curl = "0.4" 35 | env_logger = { version = "0.9", default-features = false, features = ["termcolor", "atty", "humantime"], optional = true } 36 | log = "0.4" 37 | rayon = "1" 38 | regex = "1" 39 | walkdir = "2" 40 | 41 | [features] 42 | executable = ["clap", "env_logger"] 43 | default = ["executable"] 44 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:latest AS builder 2 | COPY ./Cargo.* /app/ 3 | COPY ./src /app/src 4 | WORKDIR /app 5 | RUN cargo install --path . 6 | 7 | FROM debian:buster-slim 8 | COPY --from=builder /usr/local/cargo/bin/fixred /usr/local/bin/fixred 9 | RUN apt-get update && \ 10 | apt-get install -y --no-install-recommends libcurl4 && \ 11 | apt-get clean && rm -rf /var/lib/apt/lists/* 12 | ENTRYPOINT ["/usr/local/bin/fixred"] 13 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | the MIT License 2 | 3 | Copyright (c) 2021 rhysd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | of the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 16 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 17 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR 20 | THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | fixred 2 | ====== 3 | [![crate][crates-io-badge]][crates-io] 4 | [![CI][ci-badge]][ci] 5 | 6 | [fixred][repo] is a command line utility to fix outdated links in files with redirect URLs. 7 | 8 | demo 9 | 10 | ## Installation 11 | 12 | fixred is installed via [cargo][] package manager. [libcurl][] is necessary as dependency. 13 | 14 | ```sh 15 | cargo install fixred 16 | fixred --help 17 | ``` 18 | 19 | If you don't have Rust toolchain, [a Docker image][docker] is also available. 20 | 21 | ```sh 22 | docker run -it --rm rhysd/fixred:latest --help 23 | ``` 24 | 25 | ## Usage 26 | 27 | fixred checks redirects of URLs in text files. When a URL is redirected, fixred replaces it with the redirected one. 28 | fixred ignores invalid URLs or broken links (e.g. 404) to avoid false positives in extracted URLs. 29 | 30 | See the help output for each flags, options, and arguments. 31 | 32 | ```sh 33 | fixred --help 34 | ``` 35 | 36 | ### Fix files 37 | 38 | Most basic usage is fixing files by passing them to command line arguments. 39 | 40 | ```sh 41 | # Fix a file 42 | fixred ./docs/usage.md 43 | 44 | # Fix all files in a directory (recursively) 45 | fixred ./docs 46 | 47 | # Multiple paths can be passed 48 | fixred ./README.md ./CONTRIBUTING.md ./docs 49 | ``` 50 | 51 | Note that fixred only handles UTF8 files. Non-UTF8 files are ignored. To know which files were ignored, try `--verbose` 52 | flag. 53 | 54 | ### Fix stdin 55 | 56 | When no argument is given, fixred reads stdin and outputs result to stdout. 57 | 58 | ```sh 59 | cat ./docs/usage.md | fixred 60 | ``` 61 | 62 | ### Run via Docker container 63 | 64 | Mount local directories with `-v` and pass an environment variable (if necessary) with `-e`. Running 65 | [the Docker image][docker] executes `fixred` executable by default. 66 | 67 | ```sh 68 | # Fix all files in ./docs 69 | docker run -it --rm -v $(pwd):/app -e FIXRED_LOG=info rhysd/fixred:latest /app/docs 70 | ``` 71 | 72 | Passing the input via stdin is also possible. The result is output to stdout. 73 | 74 | ```sh 75 | # Fix stdin and output the result to stdout 76 | cat ./docs/usage.md | docker run -i --rm rhysd/fixred:latest 77 | ``` 78 | 79 | ### Redirect only once 80 | 81 | By default, fixred follows redirects repeatedly and uses the last URL to replace. However, sometimes redirecting only 82 | once would be more useful. `--shallow` flag is available for it. 83 | 84 | For example, link to raw README file in `rhysd/vim-crystal` repository (moved to `vim-crystal/vim-crystal` later) is 85 | redirected as follows. 86 | 87 | 1. https://github.com/rhysd/vim-crystal/raw/master/README.md 88 | 2. https://github.com/vim-crystal/vim-crystal/raw/master/README.md 89 | 3. https://raw.githubusercontent.com/vim-crystal/vim-crystal/master/README.md 90 | 91 | When you want to fix 1. to 2. but don't want to fix 1. to 3., `--shallow` is useful. 92 | 93 | ```sh 94 | fixred --shallow ./README.md 95 | ``` 96 | 97 | ### Filtering URLs 98 | 99 | When you want to fix only specific links in a file, filtering URLs with regular expressions is available. The following 100 | example fixes URLs which starts with `https://github.com/` using `--extract` option. 101 | 102 | ```sh 103 | fixred --extract '^https://github\.com/' ./docs 104 | ``` 105 | 106 | `--ignore` option is an invert version of `--extract`. URLs matched to the pattern are ignored. The following example 107 | avoids to resolve URLs which contain hashes. 108 | 109 | ```sh 110 | fixred --ignore '#' ./docs 111 | ``` 112 | 113 | ### Verbose logs 114 | 115 | By default, fixred outputs nothing when it runs successfully. For verbose log outputs, `--verbose` flag or `$FIXRED_LOG`i 116 | environment variable is available. 117 | 118 | ```sh 119 | # Outputs which file is being processed 120 | fixred --verbose 121 | # Or 122 | FIXRED_LOG=info fixred ./docs 123 | 124 | # Outputs what fixred is doing in detail 125 | FIXRED_LOG=debug fixred ./docs 126 | ``` 127 | 128 | ### Real-world examples 129 | 130 | - Followed the large update of GitHub document links 131 | - https://github.com/rhysd/actionlint/commit/0b7375279d2caf63203701eccc39ab091cc83a50 132 | - https://github.com/rhysd/actionlint/commit/a3f270b313affa81cc41709587cbd2588d4ac4ce 133 | - Followed the repository transition 134 | - https://github.com/benchmark-action/github-action-benchmark/commit/450cb083343edcf0fb6d82a917f890eaf2cd073f 135 | 136 | ## Use this tool as Rust library 137 | 138 | Please see [the API document][api]. And for the real world example, please see [src](./src) directory. 139 | 140 | To install as dependency, add `fixred` to your `Cargo.toml` file. Ensure to disable default features. 141 | It removes all unnecessary dependencies for using this tool as library. 142 | 143 | ```toml 144 | [dependencies] 145 | fixred = { version = "1", default-features = false, features = [] } 146 | ``` 147 | 148 | Here is a small example code 149 | 150 | ```rust 151 | use fixred::resolve::CurlResolver; 152 | use fixred::redirect::Redirector; 153 | 154 | fn main() { 155 | let red = Redirector::::default(); 156 | let fixed = red.fix(std::io::stdin(), std::io::stdout()).unwrap(); 157 | eprintln!("Fixed {} link(s)", fixed); 158 | } 159 | ``` 160 | 161 | ## License 162 | 163 | Distributed under [the MIT license](./LICENSE.txt). 164 | 165 | [repo]: https://github.com/rhysd/fixred 166 | [cargo]: https://doc.rust-lang.org/cargo/ 167 | [libcurl]: https://curl.se/libcurl/ 168 | [ci]: https://github.com/rhysd/fixred/actions/workflows/ci.yaml 169 | [ci-badge]: https://github.com/rhysd/fixred/actions/workflows/ci.yaml/badge.svg 170 | [crates-io]: https://crates.io/crates/fixred 171 | [crates-io-badge]: https://img.shields.io/crates/v/fixred.svg 172 | [docker]: https://hub.docker.com/r/rhysd/fixred 173 | [api]: https://docs.rs/fixred 174 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This is a library part of [fixred][repo] tool. 2 | //! 3 | //! To install as dependency, add `fixred` to your `Cargo.toml` file. Ensure to disable default features. 4 | //! It removes all unnecessary dependencies for using this tool as library. 5 | //! 6 | //! ```toml 7 | //! [dependencies] 8 | //! fixred = { version = "1", default-features = false, features = [] } 9 | //! ``` 10 | //! 11 | //! Here is a small example code. 12 | //! 13 | //! ``` 14 | //! use fixred::resolve::CurlResolver; 15 | //! use fixred::redirect::Redirector; 16 | //! 17 | //! let red = Redirector::::default(); 18 | //! let fixed = red.fix(std::io::stdin(), std::io::stdout()).unwrap(); 19 | //! eprintln!("Fixed {} link(s)", fixed); 20 | //! ``` 21 | //! 22 | //! For the real world example, please see [src][] directory. 23 | //! 24 | //! [repo]: https://github.com/rhysd/fixred 25 | //! [src]: https://github.com/rhysd/fixred/tree/main/src 26 | 27 | pub mod redirect; 28 | pub mod replace; 29 | pub mod resolve; 30 | pub mod url; 31 | 32 | #[cfg(test)] 33 | mod test_helper; 34 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Context, Result}; 2 | use clap::{App, Arg}; 3 | use fixred::redirect::CurlRedirector; 4 | use log::{debug, info, log_enabled, Level, LevelFilter}; 5 | use regex::Regex; 6 | use std::env; 7 | use std::io; 8 | use std::time; 9 | 10 | fn build_logger(verbose: bool) -> env_logger::Builder { 11 | let mut builder = env_logger::Builder::from_env("FIXRED_LOG"); 12 | builder.format_target(false).format_timestamp(None); 13 | if verbose && env::var_os("FIXRED_LOG").is_none() { 14 | builder.filter_level(LevelFilter::Info); 15 | } 16 | builder 17 | } 18 | 19 | fn main() -> Result<()> { 20 | let matches = App::new("fixred") 21 | .version(env!("CARGO_PKG_VERSION")) 22 | .about( 23 | "fixred is a tool to fix outdated links in text files. fixred replaces all HTTP and HTTPS \ 24 | URLs with their redirect ones. fixred ignores invalid URLs or broken links to avoid false \ 25 | positives on extracting URLs in text files.\n\n\ 26 | fixred follows redirects repeatedly and uses the last URL to replace. The behavior can be \ 27 | changed by --shallow flag to resolve the first redirect only.\n\n\ 28 | Filtering URLs to be fixed is supported. See descriptions of --extract and --ignore options.\n\n\ 29 | To enable verbose output, use --verbose flag or set $FIXRED_LOG environment variable. \ 30 | Setting --verbose or FIXRED_LOG=info outputs which file is being processed. Setting \ 31 | FIXRED_LOG=debug outputs what fixred is doing.\n\n\ 32 | Note that fixred only handles UTF8 files. Non-UTF8 files are ignored.\n\n\ 33 | Visit https://github.com/rhysd/fixred#usage for more details with several examples.", 34 | ) 35 | .arg( 36 | Arg::new("shallow") 37 | .short('s') 38 | .long("shallow") 39 | .about("Redirect only once when resolving a URL redirect") 40 | ) 41 | .arg( 42 | Arg::new("extract") 43 | .short('e') 44 | .long("extract") 45 | .takes_value(true) 46 | .value_name("REGEX") 47 | .about("Fix URLs which are matched to this pattern"), 48 | ) 49 | .arg( 50 | Arg::new("ignore") 51 | .short('r') 52 | .long("ignore") 53 | .takes_value(true) 54 | .value_name("REGEX") 55 | .about("Fix URLs which are NOT matched to this pattern"), 56 | ) 57 | .arg( 58 | Arg::new("PATH") 59 | .about( 60 | "Directory or file path to fix. When a directory path is given, all files in it \ 61 | are fixed recursively. When no path is given, fixred reads input from stdin and \ 62 | outputs the result to stdout", 63 | ) 64 | .multiple_values(true), 65 | ) 66 | .arg( 67 | Arg::new("verbose") 68 | .short('v') 69 | .long("verbose") 70 | .about("Output verbose log. This is the same as setting \"info\" to $FIXRED_LOG environment variable") 71 | ) 72 | .get_matches(); 73 | 74 | build_logger(matches.is_present("verbose")).init(); 75 | 76 | let start = log_enabled!(Level::Debug).then(time::Instant::now); 77 | 78 | let red = CurlRedirector::default() 79 | .extract(matches.value_of("extract").map(Regex::new).transpose()?) 80 | .ignore(matches.value_of("ignore").map(Regex::new).transpose()?) 81 | .shallow(matches.is_present("shallow")); 82 | 83 | if let Some(paths) = matches.values_of_os("PATH") { 84 | info!("Processing all files in given paths via command line arguments"); 85 | let count = red.fix_all_files(paths)?; 86 | info!("Processed {} files", count); 87 | } else { 88 | info!("Fixing redirects in stdin"); 89 | let stdin = io::stdin(); 90 | let stdout = io::stdout(); 91 | let count = red 92 | .fix(stdin.lock(), stdout.lock()) 93 | .context("While processing stdin")?; 94 | info!("Fixed {} links in stdin", count); 95 | } 96 | 97 | if let Some(start) = start { 98 | let secs = time::Instant::now().duration_since(start).as_secs_f32(); 99 | debug!("Elapsed: {} seconds", secs); 100 | } 101 | 102 | Ok(()) 103 | } 104 | -------------------------------------------------------------------------------- /src/redirect.rs: -------------------------------------------------------------------------------- 1 | use crate::replace::{replace_all, Replacement}; 2 | use crate::resolve::{CurlResolver, Resolver}; 3 | use crate::url::find_all_urls; 4 | use anyhow::{Context, Result}; 5 | use log::{debug, info, warn}; 6 | use rayon::prelude::*; 7 | use regex::Regex; 8 | use std::ffi::OsStr; 9 | use std::fs; 10 | use std::io::{BufWriter, Read, Write}; 11 | use std::path::Path; 12 | use walkdir::WalkDir; 13 | 14 | #[derive(Default)] 15 | pub struct Redirector { 16 | extract: Option, 17 | ignore: Option, 18 | resolver: R, 19 | } 20 | 21 | impl Redirector { 22 | pub fn extract(mut self, pattern: Option) -> Self { 23 | debug!("Regex to extract URLs: {:?}", pattern); 24 | self.extract = pattern; 25 | self 26 | } 27 | 28 | pub fn ignore(mut self, pattern: Option) -> Self { 29 | debug!("Regex to ignore URLs: {:?}", pattern); 30 | self.ignore = pattern; 31 | self 32 | } 33 | 34 | pub fn shallow(mut self, enabled: bool) -> Self { 35 | debug!("Shallow redirect?: {}", enabled); 36 | self.resolver.shallow(enabled); 37 | self 38 | } 39 | 40 | fn should_resolve(&self, url: &str) -> bool { 41 | if let Some(r) = &self.extract { 42 | if !r.is_match(url) { 43 | return false; 44 | } 45 | } 46 | if let Some(r) = &self.ignore { 47 | if r.is_match(url) { 48 | return false; 49 | } 50 | } 51 | true 52 | } 53 | 54 | fn find_replacements(&self, content: &str) -> Vec { 55 | let spans = find_all_urls(content); // Collect to Vec to use par_iter which is more efficient than par_bridge 56 | debug!("Found {} links", spans.len()); 57 | let replacements = spans 58 | .into_par_iter() 59 | .filter_map(|(start, end)| { 60 | let url = &content[start..end]; 61 | if !self.should_resolve(url) { 62 | debug!("Skipped URL: {}", url); 63 | return None; 64 | } 65 | let url = self.resolver.resolve(url); 66 | url.map(|text| Replacement { start, end, text }) 67 | }) 68 | .collect::>(); // Collect to Vec to check errors before overwriting files 69 | debug!("Found {} redirects", replacements.len()); 70 | replacements 71 | } 72 | 73 | pub fn fix_file(&self, file: &Path) -> Result<()> { 74 | info!("Fixing redirects in {:?}", &file); 75 | 76 | let content = match fs::read_to_string(&file) { 77 | Err(err) => { 78 | warn!("Ignored non-UTF8 file {:?}: {}", &file, err); 79 | return Ok(()); 80 | } 81 | Ok(s) => s, 82 | }; 83 | let replacements = self.find_replacements(&content); 84 | if replacements.is_empty() { 85 | info!("Fixed no link in {:?} (skipped overwriting)", &file); 86 | return Ok(()); 87 | } 88 | let mut out = BufWriter::new(fs::File::create(&file)?); // Truncate the file after all replacements are collected without error 89 | replace_all(&mut out, &content, &replacements)?; 90 | 91 | info!("Fixed {} links in {:?}", replacements.len(), &file); 92 | Ok(()) 93 | } 94 | 95 | pub fn fix_all_files<'a>(&self, paths: impl Iterator) -> Result { 96 | paths 97 | .flat_map(WalkDir::new) 98 | .filter(|entry| match entry { 99 | Ok(e) => e.file_type().is_file(), 100 | Err(_) => true, 101 | }) 102 | .map(|entry| { 103 | let path = entry?.into_path(); 104 | self.fix_file(&path) 105 | .with_context(|| format!("While processing {:?}", &path))?; 106 | Ok(1) 107 | }) 108 | .sum() 109 | } 110 | 111 | pub fn fix(&self, mut input: T, output: U) -> Result { 112 | let mut content = String::new(); 113 | input.read_to_string(&mut content)?; 114 | let content = &content; 115 | let replacements = self.find_replacements(content); 116 | replace_all(output, content, &replacements)?; 117 | Ok(replacements.len()) 118 | } 119 | } 120 | 121 | pub type CurlRedirector = Redirector; 122 | 123 | #[cfg(test)] 124 | mod tests { 125 | use super::*; 126 | use crate::test_helper::*; 127 | use std::iter; 128 | use std::path::PathBuf; 129 | 130 | type TestRedirector = Redirector; 131 | 132 | #[test] 133 | fn fix_all_files_recursively() { 134 | // Tests with actual file system 135 | 136 | let entries = &[ 137 | TestDirEntry::File( 138 | "test1.txt", 139 | "https://foo1.example.com\nhttps://example.com/foo1\nhttps://example.com\n", 140 | ), 141 | TestDirEntry::Dir("dir1"), 142 | TestDirEntry::File( 143 | "dir1/test2.txt", 144 | "https://foo2.example.com\nhttps://example.com/foo2\nhttps://example.com\n", 145 | ), 146 | TestDirEntry::File( 147 | "dir1/test3.txt", 148 | "https://foo3.example.com\nhttps://example.com/foo3\nhttps://example.com\n", 149 | ), 150 | TestDirEntry::Dir("dir1/dir2"), 151 | TestDirEntry::File( 152 | "dir1/dir2/test4.txt", 153 | "https://foo4.example.com\nhttps://example.com/foo4\nhttps://example.com\n", 154 | ), 155 | TestDirEntry::File( 156 | "dir1/dir2/test5.txt", 157 | "https://foo5.example.com\nhttps://example.com/foo5\nhttps://example.com\n", 158 | ), 159 | ]; 160 | 161 | let dir = TestDir::new(entries).unwrap(); 162 | 163 | let red = TestRedirector::default(); 164 | let root = &dir.root; 165 | let paths = &[root.join("test1.txt"), root.join("dir1")]; 166 | let count = red.fix_all_files(paths.iter().map(|p| p.as_ref())).unwrap(); 167 | assert_eq!(count, dir.files.len()); 168 | 169 | let want: Vec<_> = dir 170 | .files 171 | .iter() 172 | .map(|(p, c)| (p.clone(), c.replace("foo", "piyo"))) 173 | .collect(); 174 | assert_files(&want); 175 | } 176 | 177 | #[test] 178 | fn ignore_non_utf8_file() { 179 | // Invalid UTF-8 sequence 180 | let content = b"\xf0\x28\x8c\xbc"; 181 | std::str::from_utf8(content).unwrap_err(); 182 | 183 | let entries = &[TestDirEntry::Binary("test.bin", content)]; 184 | let dir = TestDir::new(entries).unwrap(); 185 | 186 | let red = TestRedirector::default(); 187 | let path = dir.root.join("test.bin"); 188 | let count = red.fix_all_files(iter::once(path.as_ref())).unwrap(); 189 | assert_eq!(count, 1); 190 | } 191 | 192 | #[test] 193 | fn read_file_error() { 194 | let red = TestRedirector::default(); 195 | let mut p = PathBuf::new(); 196 | p.push("this-file"); 197 | p.push("does-not"); 198 | p.push("exist.txt"); 199 | red.fix_all_files(iter::once(p.as_ref())).unwrap_err(); 200 | } 201 | 202 | #[test] 203 | fn fix_reader_writer() { 204 | let mut output = vec![]; 205 | let input = " 206 | this is test https://foo1.example.com 207 | https://example.com/foo1 208 | https://example.com 209 | done."; 210 | 211 | let red = TestRedirector::default(); 212 | let fixed = red.fix(input.as_bytes(), &mut output).unwrap(); 213 | assert_eq!(fixed, 2); 214 | 215 | let want = input.replace("foo", "piyo"); 216 | let have = String::from_utf8(output).unwrap(); 217 | assert_eq!(want, have); 218 | } 219 | 220 | #[test] 221 | fn fix_shallow_redirect() { 222 | let mut output = vec![]; 223 | let input = " 224 | this is test https://foo1.example.com 225 | https://example.com/foo1 226 | https://example.com 227 | done."; 228 | 229 | let red = TestRedirector::default().shallow(true); 230 | let fixed = red.fix(input.as_bytes(), &mut output).unwrap(); 231 | assert_eq!(fixed, 2); 232 | 233 | let want = input.replace("foo", "bar"); 234 | let have = String::from_utf8(output).unwrap(); 235 | assert_eq!(want, have); 236 | } 237 | 238 | #[test] 239 | fn exract_urls() { 240 | let mut output = vec![]; 241 | let input = " 242 | - https://github.com/rhysd/foo 243 | - https://rhysd.github.io/foo 244 | - https://docs.github.com/foo/some-docs 245 | - https://foo.example.com/foo 246 | "; 247 | 248 | let pat = Regex::new("github\\.com/").unwrap(); 249 | let red = TestRedirector::default().extract(Some(pat)); 250 | let fixed = red.fix(input.as_bytes(), &mut output).unwrap(); 251 | assert_eq!(fixed, 2); 252 | 253 | let want = input 254 | .replace("github.com/rhysd/foo", "github.com/rhysd/piyo") 255 | .replace("docs.github.com/foo", "docs.github.com/piyo"); 256 | let have = String::from_utf8(output).unwrap(); 257 | assert_eq!(want, have); 258 | } 259 | 260 | #[test] 261 | fn ignore_urls() { 262 | let mut output = vec![]; 263 | let input = " 264 | - https://github.com/rhysd/foo 265 | - https://rhysd.github.io/foo 266 | - https://docs.github.com/foo/some-docs 267 | - https://foo.example.com/foo 268 | "; 269 | 270 | let pat = Regex::new("github\\.com/").unwrap(); 271 | let red = TestRedirector::default().ignore(Some(pat)); 272 | let fixed = red.fix(input.as_bytes(), &mut output).unwrap(); 273 | assert_eq!(fixed, 2); 274 | 275 | let want = input 276 | .replace("rhysd.github.io/foo", "rhysd.github.io/piyo") 277 | .replace("foo.example.com/foo", "piyo.example.com/piyo"); 278 | let have = String::from_utf8(output).unwrap(); 279 | assert_eq!(want, have); 280 | } 281 | 282 | #[test] 283 | fn extract_and_ignore_urls() { 284 | let mut output = vec![]; 285 | let input = " 286 | - https://github.com/rhysd/foo 287 | - https://rhysd.github.io/foo 288 | - https://docs.github.com/foo/some-docs 289 | - https://foo.example.com/foo 290 | "; 291 | 292 | let pat1 = Regex::new("example\\.com/").unwrap(); 293 | let pat2 = Regex::new("github\\.com/").unwrap(); 294 | let red = TestRedirector::default() 295 | .extract(Some(pat1)) 296 | .ignore(Some(pat2)); 297 | let fixed = red.fix(input.as_bytes(), &mut output).unwrap(); 298 | assert_eq!(fixed, 1); 299 | 300 | let want = input.replace("foo.example.com/foo", "piyo.example.com/piyo"); 301 | let have = String::from_utf8(output).unwrap(); 302 | assert_eq!(want, have); 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /src/replace.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::io::Write; 3 | 4 | pub struct Replacement { 5 | pub start: usize, 6 | pub end: usize, 7 | pub text: String, 8 | } 9 | 10 | impl Replacement { 11 | pub fn new(start: usize, end: usize, text: impl Into) -> Replacement { 12 | let text = text.into(); 13 | Replacement { start, end, text } 14 | } 15 | } 16 | 17 | pub fn replace_all(mut out: W, input: &str, replacements: &[Replacement]) -> Result<()> { 18 | let mut i = 0; 19 | for replacement in replacements.iter() { 20 | let Replacement { start, end, text } = replacement; 21 | out.write_all(input[i..*start].as_bytes())?; 22 | out.write_all(text.as_bytes())?; 23 | i = *end; 24 | } 25 | out.write_all(input[i..].as_bytes())?; 26 | Ok(out.flush()?) 27 | } 28 | 29 | #[cfg(test)] 30 | mod tests { 31 | use super::*; 32 | use crate::test_helper::*; 33 | use std::array::IntoIter; 34 | use std::str; 35 | 36 | #[test] 37 | fn replace_one() { 38 | let mut buf = Vec::new(); 39 | let rep = &[Replacement::new(4, 4 + "hello".len(), "goodbye")]; 40 | replace_all(&mut buf, "hi! hello world!", rep).unwrap(); 41 | let o = str::from_utf8(&buf).unwrap(); 42 | assert_eq!(o, "hi! goodbye world!"); 43 | } 44 | 45 | #[test] 46 | fn replace_multiple() { 47 | let mut buf = Vec::new(); 48 | let rep = &[ 49 | Replacement::new(0, "hi!".len(), "woo!"), 50 | Replacement::new(4, 4 + "hello".len(), "goodbye"), 51 | Replacement::new(10, 10 + "world".len(), "universe"), 52 | ]; 53 | replace_all(&mut buf, "hi! hello world!", rep).unwrap(); 54 | let o = str::from_utf8(&buf).unwrap(); 55 | assert_eq!(o, "woo! goodbye universe!"); 56 | } 57 | 58 | #[test] 59 | fn replace_entire() { 60 | let mut buf = Vec::new(); 61 | let rep = &[Replacement::new(0, "hello".len(), "goodbye")]; 62 | replace_all(&mut buf, "hello", rep).unwrap(); 63 | let o = str::from_utf8(&buf).unwrap(); 64 | assert_eq!(o, "goodbye"); 65 | } 66 | 67 | #[test] 68 | fn no_replacement() { 69 | for i in IntoIter::new(["", "foo"]) { 70 | let mut buf = Vec::new(); 71 | replace_all(&mut buf, i, &[]).unwrap(); 72 | let o = str::from_utf8(&buf).unwrap(); 73 | assert_eq!(i, o); 74 | } 75 | } 76 | 77 | #[test] 78 | fn write_error() { 79 | assert!(replace_all(WriteErrorWriter, "foo", &[]).is_err()); 80 | } 81 | 82 | #[test] 83 | fn flush_error() { 84 | assert!(replace_all(FlushErrorWriter, "foo", &[]).is_err()); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/resolve.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use chashmap::CHashMap; 3 | use curl::easy::Easy; 4 | use log::{debug, warn}; 5 | 6 | pub trait Resolver: Default + Sync { 7 | fn shallow(&mut self, b: bool); 8 | fn resolve(&self, url: &str) -> Option; 9 | } 10 | 11 | #[derive(Default)] 12 | pub struct CurlResolver { 13 | shallow: bool, 14 | cache: CHashMap>, 15 | } 16 | 17 | impl CurlResolver { 18 | fn try_resolve(&self, url: &str) -> Result> { 19 | debug!("Resolving {}", url); 20 | 21 | if let Some(u) = self.cache.get(url) { 22 | debug!("Cache hit: {} -> {:?}", url, *u); 23 | return Ok(u.clone()); 24 | } 25 | 26 | // https://datatracker.ietf.org/doc/html/rfc3986#section-3 27 | let fragment = url.find('#').map(|i| &url[i + 1..]); 28 | 29 | debug!("Sending HEAD request to {}", url); 30 | let mut curl = Easy::new(); 31 | curl.nobody(true)?; 32 | curl.url(url)?; 33 | let resolved = if self.shallow { 34 | curl.perform()?; 35 | curl.redirect_url()? // Get the first redirect URL 36 | } else { 37 | curl.follow_location(true)?; 38 | curl.perform()?; 39 | curl.effective_url()? 40 | }; 41 | let red = resolved.and_then(|u| { 42 | (u != url).then(|| { 43 | if let Some(fragment) = fragment { 44 | format!("{}#{}", u, fragment) 45 | } else { 46 | u.to_string() 47 | } 48 | }) 49 | }); 50 | debug!("Resolved redirect: {} -> {:?}", url, red); 51 | self.cache.insert(url.to_string(), red.clone()); 52 | Ok(red) 53 | } 54 | } 55 | 56 | impl Resolver for CurlResolver { 57 | fn shallow(&mut self, enabled: bool) { 58 | self.shallow = enabled; 59 | } 60 | 61 | fn resolve(&self, url: &str) -> Option { 62 | // Do not return error on resolving URLs because it is normal case that broken URL is passed to this function. 63 | match self.try_resolve(url) { 64 | Ok(ret) => ret, 65 | Err(err) => { 66 | warn!("Could not resolve {:?}: {}", url, err); 67 | None 68 | } 69 | } 70 | } 71 | } 72 | 73 | #[cfg(test)] 74 | mod tests { 75 | use super::*; 76 | 77 | #[test] 78 | fn resolve_url_with_cache() { 79 | // Redirect: github.com/rhysd/ -> github.com/vim-crystal/ -> raw.githubusercontent 80 | let url = "https://github.com/rhysd/vim-crystal/raw/master/README.md"; 81 | 82 | let res = CurlResolver::default(); 83 | let resolved = res.try_resolve(url).unwrap(); 84 | let resolved = resolved.unwrap(); 85 | assert!( 86 | resolved.starts_with("https://raw.githubusercontent.com/vim-crystal/"), 87 | "URL: {}", 88 | resolved 89 | ); 90 | 91 | assert_eq!(*res.cache.get(url).unwrap(), Some(resolved.clone())); 92 | 93 | let cached = res.try_resolve(url).unwrap(); 94 | assert_eq!(resolved, cached.unwrap()); 95 | } 96 | 97 | #[test] 98 | fn resolve_shallow_redirect() { 99 | // Redirect: github.com/rhysd/ -> github.com/vim-crystal/ -> raw.githubusercontent 100 | let url = "https://github.com/rhysd/vim-crystal/raw/master/README.md"; 101 | 102 | let mut res = CurlResolver::default(); 103 | res.shallow(true); 104 | let resolved = res.try_resolve(url).unwrap(); 105 | let resolved = resolved.unwrap(); 106 | assert!( 107 | resolved.starts_with("https://github.com/vim-crystal/vim-crystal/"), 108 | "URL: {}", 109 | resolved 110 | ); 111 | } 112 | 113 | #[test] 114 | fn resolve_url_not_found() { 115 | // Redirect: github.com/rhysd/ -> github.com/vim-crystal/ -> raw.githubusercontent 116 | let url = "https://github.com/rhysd/this-repo-does-not-exist"; 117 | 118 | let res = CurlResolver::default(); 119 | let resolved = res.resolve(url); 120 | assert_eq!(resolved, None); 121 | 122 | assert_eq!(*res.cache.get(url).unwrap(), None); 123 | 124 | let cached = res.resolve(url); 125 | assert_eq!(resolved, cached); 126 | } 127 | 128 | #[test] 129 | fn resolve_url_with_fragment() { 130 | // Redirect: github.com/rhysd/ -> github.com/vim-crystal 131 | let url = "https://github.com/rhysd/vim-crystal#readme"; 132 | 133 | let res = CurlResolver::default(); 134 | let resolved = res.resolve(url).unwrap(); 135 | assert!( 136 | resolved.ends_with("/vim-crystal#readme"), 137 | "URL: {}", 138 | resolved 139 | ); 140 | } 141 | 142 | #[test] 143 | fn url_parse_error() { 144 | let res = CurlResolver::default(); 145 | let resolved = res.try_resolve("https://"); 146 | assert!(resolved.is_err(), "{:?}", resolved); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/test_helper.rs: -------------------------------------------------------------------------------- 1 | use crate::resolve::Resolver; 2 | use std::env; 3 | use std::fs; 4 | use std::io::{Error, ErrorKind, Result, Write}; 5 | use std::path::{Path, PathBuf}; 6 | use std::sync::atomic::{AtomicUsize, Ordering}; 7 | use std::time; 8 | 9 | pub(crate) struct WriteErrorWriter; 10 | impl Write for WriteErrorWriter { 11 | fn write(&mut self, _buf: &[u8]) -> Result { 12 | Err(Error::new(ErrorKind::Other, "test")) 13 | } 14 | fn flush(&mut self) -> Result<()> { 15 | Ok(()) 16 | } 17 | } 18 | 19 | pub(crate) struct FlushErrorWriter; 20 | impl Write for FlushErrorWriter { 21 | fn write(&mut self, buf: &[u8]) -> Result { 22 | Ok(buf.len()) 23 | } 24 | fn flush(&mut self) -> Result<()> { 25 | Err(Error::new(ErrorKind::Other, "test")) 26 | } 27 | } 28 | 29 | pub(crate) enum TestDirEntry<'a> { 30 | Dir(&'a str), 31 | File(&'a str, &'a str), 32 | Binary(&'a str, &'a [u8]), 33 | } 34 | 35 | impl<'a> TestDirEntry<'a> { 36 | fn path(&self, root: &Path) -> PathBuf { 37 | let mut path = root.to_owned(); 38 | let p = match self { 39 | Self::Dir(p) => p, 40 | Self::File(p, _) => p, 41 | Self::Binary(p, _) => p, 42 | }; 43 | for name in p.split('/').filter(|n| !n.is_empty()) { 44 | path.push(name); 45 | } 46 | path 47 | } 48 | 49 | fn create(&self, root: &Path) -> Result> { 50 | let path = self.path(root); 51 | match self { 52 | TestDirEntry::Dir(_) => { 53 | fs::create_dir(&path)?; 54 | Ok(None) 55 | } 56 | TestDirEntry::File(_, content) => { 57 | fs::write(&path, content.as_bytes())?; 58 | Ok(Some((path, content.to_string()))) 59 | } 60 | TestDirEntry::Binary(_, content) => { 61 | fs::write(&path, content)?; 62 | Ok(None) 63 | } 64 | } 65 | } 66 | } 67 | 68 | // Since tests are run in parallel, test directory name must contains unique ID to avoid name conflict. 69 | static TEST_DIR_ID: AtomicUsize = AtomicUsize::new(0); 70 | 71 | pub(crate) struct TestDir { 72 | pub root: PathBuf, 73 | pub files: Vec<(PathBuf, String)>, 74 | } 75 | 76 | impl TestDir { 77 | pub fn new<'a>(iter: impl IntoIterator>) -> Result { 78 | let mut root = env::temp_dir(); 79 | let root_name = format!( 80 | "redfix-test-{}-{}", 81 | TEST_DIR_ID.fetch_add(1, Ordering::Relaxed), 82 | time::SystemTime::now() 83 | .duration_since(time::SystemTime::UNIX_EPOCH) 84 | .unwrap() 85 | .as_millis() 86 | ); 87 | root.push(root_name); 88 | fs::create_dir(&root)?; 89 | let mut dir = TestDir { 90 | root, 91 | files: vec![], 92 | }; 93 | for p in iter { 94 | if let Some((p, c)) = p.create(&dir.root)? { 95 | dir.files.push((p, c)); 96 | } 97 | } 98 | Ok(dir) 99 | } 100 | 101 | pub fn delete(&self) -> Result<()> { 102 | fs::remove_dir_all(&self.root) 103 | } 104 | } 105 | 106 | impl Drop for TestDir { 107 | fn drop(&mut self) { 108 | self.delete().unwrap(); 109 | } 110 | } 111 | 112 | pub(crate) fn assert_files(files: &[(PathBuf, String)]) { 113 | for (path, want) in files { 114 | let have = fs::read_to_string(path).unwrap(); 115 | assert_eq!(want, &have, "content in {:?} mismatched", path); 116 | } 117 | } 118 | 119 | // redirect foo -> bar -> piyo 120 | #[derive(Default)] 121 | pub(crate) struct FooToPiyoResolver { 122 | pub shallow: bool, 123 | } 124 | 125 | impl Resolver for FooToPiyoResolver { 126 | fn shallow(&mut self, b: bool) { 127 | self.shallow = b; 128 | } 129 | fn resolve(&self, url: &str) -> Option { 130 | let to = if self.shallow { "bar" } else { "piyo" }; 131 | let new = url.replace("foo", to); 132 | (url != new).then(|| new) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/url.rs: -------------------------------------------------------------------------------- 1 | use aho_corasick::AhoCorasick; 2 | 3 | enum Char { 4 | Invalid, 5 | Term, 6 | NonTerm, 7 | } 8 | 9 | // https://datatracker.ietf.org/doc/html/rfc3986#section-2 10 | // > unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" 11 | // > reserved = gen-delims / sub-delims 12 | // > gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" 13 | // > sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" 14 | // > pct-encoded = "%" HEXDIG HEXDIG 15 | fn url_char_kind(c: char) -> Char { 16 | match c { 17 | c if c.is_alphanumeric() => Char::Term, 18 | '.' | ':' | '?' | '#' | '[' | ']' | '@' | '!' | '$' | '&' | '\'' | '(' | ')' | '*' 19 | | '+' | ',' | ';' | '%' => Char::NonTerm, 20 | '-' | '_' | '~' | '/' | '=' => Char::Term, 21 | _ => Char::Invalid, 22 | } 23 | } 24 | 25 | pub fn find_all_urls(content: &str) -> Vec<(usize, usize)> { 26 | AhoCorasick::new(&["https://", "http://"]) 27 | .find_iter(content) 28 | .filter_map(|m| { 29 | let start = m.start(); 30 | let end = m.end(); 31 | 32 | let mut idx = 0; 33 | for (i, c) in content[end..].char_indices() { 34 | match url_char_kind(c) { 35 | Char::NonTerm => {} 36 | Char::Term => { 37 | // Since range is [start, end), idx should be index of the next character 38 | idx = i + c.len_utf8(); 39 | } 40 | Char::Invalid => break, 41 | } 42 | } 43 | if idx == 0 { 44 | None 45 | } else { 46 | let end = end + idx; 47 | Some((start, end)) 48 | } 49 | }) 50 | .collect() 51 | } 52 | 53 | #[cfg(test)] 54 | mod tests { 55 | use super::*; 56 | 57 | #[test] 58 | fn empty() { 59 | let v = find_all_urls(""); 60 | assert!(v.is_empty()); 61 | } 62 | 63 | #[test] 64 | fn no_url() { 65 | let v = find_all_urls("foo bar baz"); 66 | assert!(v.is_empty()); 67 | } 68 | 69 | #[test] 70 | fn empty_after_scheme() { 71 | let v = find_all_urls("contains(s, 'https://')"); 72 | assert!(v.is_empty()); 73 | } 74 | 75 | #[test] 76 | fn entire_url() { 77 | let s = "http://example.com"; 78 | assert_eq!(find_all_urls(s), &[(0, s.len())]); 79 | 80 | let s = "https://example.com"; 81 | assert_eq!(find_all_urls(s), &[(0, s.len())]); 82 | } 83 | 84 | #[test] 85 | fn url_in_sentence() { 86 | let s = "the URL is https://example.com."; 87 | let (b, e) = find_all_urls(s)[0]; 88 | assert_eq!(&s[b..e], "https://example.com"); 89 | 90 | let s = "the URL is https://example.com!"; 91 | let (b, e) = find_all_urls(s)[0]; 92 | assert_eq!(&s[b..e], "https://example.com"); 93 | 94 | let s = "the URL is [the link](https://example.com)"; 95 | let (b, e) = find_all_urls(s)[0]; 96 | assert_eq!(&s[b..e], "https://example.com"); 97 | } 98 | 99 | #[test] 100 | fn url_ends_with_slash() { 101 | let s = "the GitHub URL is https://github.com/, check it out"; 102 | let (b, e) = find_all_urls(s)[0]; 103 | assert_eq!(&s[b..e], "https://github.com/"); 104 | } 105 | 106 | #[test] 107 | fn percent_encoding() { 108 | let s = "https://example.com/?foo=%E3%81%82%E3%81%84%E3%81%86%E3%81%88%E3%81%8A&bar=true"; 109 | let t = format!("see the URL {} for more details", s); 110 | let (b, e) = find_all_urls(&t)[0]; 111 | assert_eq!(&t[b..e], s); 112 | } 113 | 114 | #[test] 115 | fn multiple_urls() { 116 | let s = " 117 | - Repository: https://github.com/rhysd/actionlint 118 | - Playground: https://rhysd.github.io/actionlint/ 119 | - GitHub Actions official documentations 120 | - Workflow syntax: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions 121 | - Expression syntax: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions 122 | - Built-in functions: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#functions 123 | - Webhook events: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#webhook-events 124 | - Self-hosted runner: https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners 125 | - Security: https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions 126 | - CRON syntax: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 127 | - shellcheck: https://github.com/koalaman/shellcheck 128 | - pyflakes: https://github.com/PyCQA/pyflakes 129 | - Japanese blog posts 130 | - GitHub Actions のワークフローをチェックする actionlint をつくった: https://rhysd.hatenablog.com/entry/2021/07/11/214313 131 | - actionlint v1.4 → v1.6 で実装した新機能の紹介: https://rhysd.hatenablog.com/entry/2021/08/11/221044 132 | "; 133 | 134 | let want = &[ 135 | "https://github.com/rhysd/actionlint", 136 | "https://rhysd.github.io/actionlint/", 137 | "https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions", 138 | "https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions", 139 | "https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#functions", 140 | "https://docs.github.com/en/actions/reference/events-that-trigger-workflows#webhook-events", 141 | "https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners", 142 | "https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions", 143 | "https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07", 144 | "https://github.com/koalaman/shellcheck", 145 | "https://github.com/PyCQA/pyflakes", 146 | "https://rhysd.hatenablog.com/entry/2021/07/11/214313", 147 | "https://rhysd.hatenablog.com/entry/2021/08/11/221044", 148 | ]; 149 | 150 | let have: Vec<_> = find_all_urls(s) 151 | .into_iter() 152 | .map(|(b, e)| &s[b..e]) 153 | .collect(); 154 | 155 | assert_eq!(have, want); 156 | } 157 | } 158 | --------------------------------------------------------------------------------