├── .all-contributorsrc ├── .cargo └── config.toml ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── actions-rs │ └── grcov.yml ├── dependabot.yml ├── pull_request_template.md ├── stale.yml └── workflows │ ├── build.yml │ ├── check.yml │ ├── cicd-to-dockerhub.yml │ ├── coverage.yml │ └── winget.yml ├── .gitignore ├── .rustfmt.toml ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── Makefile ├── Makefile.toml ├── README.md ├── build.rs ├── choco_package ├── feroxbuster.nuspec ├── legal │ ├── LICENSE.txt │ └── VERIFICATION.txt └── tools │ ├── chocolateyinstall.ps1 │ └── chocolateyuninstall.ps1 ├── docs ├── .nojekyll └── index.html ├── ferox-config.toml.example ├── img ├── auto-bail-demo.gif ├── auto-tune-demo.gif ├── cancel-menu.png ├── cancel-scan.gif ├── demo.gif ├── dir-scan-bar-explained.png ├── extract-scan-cmp-normal.gif ├── insecure.png ├── limit-demo.gif ├── logo │ ├── default-cropped.png │ └── logo.png ├── normal-scan-cmp-extract.gif ├── pause-resume-demo.gif ├── rate-limit-demo.gif ├── replay-proxy-demo.gif ├── response-bar-explained.png ├── resumed-scan.gif ├── save-state.png ├── small-term.png ├── time-limit.gif └── total-bar-explained.png ├── install-nix.sh ├── shell_completions ├── _feroxbuster ├── _feroxbuster.ps1 ├── feroxbuster.bash ├── feroxbuster.elv └── feroxbuster.fish ├── snapcraft.yaml ├── src ├── banner │ ├── container.rs │ ├── entry.rs │ ├── mod.rs │ └── tests.rs ├── client.rs ├── config │ ├── container.rs │ ├── mod.rs │ ├── tests.rs │ └── utils.rs ├── event_handlers │ ├── command.rs │ ├── container.rs │ ├── filters.rs │ ├── inputs.rs │ ├── mod.rs │ ├── outputs.rs │ ├── scans.rs │ └── statistics.rs ├── extractor │ ├── builder.rs │ ├── container.rs │ ├── mod.rs │ └── tests.rs ├── filters │ ├── container.rs │ ├── empty.rs │ ├── init.rs │ ├── lines.rs │ ├── mod.rs │ ├── regex.rs │ ├── similarity.rs │ ├── size.rs │ ├── status_code.rs │ ├── tests.rs │ ├── utils.rs │ ├── wildcard.rs │ └── words.rs ├── heuristics.rs ├── lib.rs ├── logger.rs ├── macros.rs ├── main.rs ├── message.rs ├── nlp │ ├── constants.rs │ ├── document.rs │ ├── mod.rs │ ├── model.rs │ ├── term.rs │ └── utils.rs ├── parser.rs ├── progress.rs ├── response.rs ├── scan_manager │ ├── menu.rs │ ├── mod.rs │ ├── order.rs │ ├── response_container.rs │ ├── scan.rs │ ├── scan_container.rs │ ├── state.rs │ ├── tests.rs │ └── utils.rs ├── scanner │ ├── ferox_scanner.rs │ ├── init.rs │ ├── limit_heap.rs │ ├── mod.rs │ ├── policy_data.rs │ ├── requester.rs │ ├── tests.rs │ └── utils.rs ├── statistics │ ├── container.rs │ ├── error.rs │ ├── field.rs │ ├── init.rs │ ├── macros.rs │ ├── mod.rs │ └── tests.rs ├── traits.rs ├── url.rs └── utils.rs └── tests ├── extra-words ├── mutual-auth ├── Caddyfile ├── README.md ├── certs │ ├── client │ │ ├── client.crt │ │ └── client.key │ └── server │ │ ├── ca.crt │ │ ├── server.crt │ │ ├── server.crt.1 │ │ ├── server.crt.2 │ │ ├── server.der │ │ └── server.key └── gen-certs.sh ├── policy-test-words.shuffled ├── test_banner.rs ├── test_config.rs ├── test_deny_list.rs ├── test_extractor.rs ├── test_filters.rs ├── test_heuristics.rs ├── test_main.rs ├── test_parser.rs ├── test_policies.rs ├── test_scan_manager.rs ├── test_scanner.rs └── utils └── mod.rs /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.armv7-unknown-linux-gnueabihf] 2 | linker = "arm-linux-gnueabihf-gcc" 3 | 4 | [target.aarch64-unknown-linux-gnu] 5 | linker = "aarch64-linux-gnu-gcc" 6 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [epi052] 4 | ko_fi: epi052 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG] " 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. 16 | 2. 17 | 3. 18 | 19 | **Expected behavior** 20 | A clear and concise description of what you expected to happen. 21 | 22 | **Traceback / Error Output** 23 | If applicable, add error output to help explain your problem. 24 | 25 | **Environment (please complete the following information):** 26 | - feroxbuster version: [e.g. v1.0.1] 27 | - OS [e.g. ubuntu 20.04] 28 | 29 | **Additional context** 30 | Add any other context about the problem here. 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE REQUEST] " 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/actions-rs/grcov.yml: -------------------------------------------------------------------------------- 1 | branch: false 2 | ignore-not-existing: true 3 | llvm: true 4 | output-type: lcov 5 | output-path: ./lcov.info 6 | # excl-br-line: "^\\s*((debug_)?assert(_eq|_ne)?!|#\\[derive\\(|log::)" 7 | ignore: 8 | - "../*" 9 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: cargo 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Landing a Pull Request (PR) 2 | 3 | Long form explanations of most of the items below can be found in the [CONTRIBUTING](https://github.com/epi052/feroxbuster/blob/master/CONTRIBUTING.md) guide. 4 | 5 | ## Branching checklist 6 | - [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one) 7 | - [ ] Your PR description references the associated issue (i.e. fixes #123456) 8 | - [ ] Code is in its own branch 9 | - [ ] Branch name is related to the PR contents 10 | - [ ] PR targets main 11 | 12 | ## Static analysis checks 13 | - [ ] All rust files are formatted using `cargo fmt` 14 | - [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic` 15 | - [ ] All existing tests pass 16 | 17 | ## Documentation 18 | - [ ] New code is documented using [doc comments](https://doc.rust-lang.org/stable/rust-by-example/meta/doc.html) 19 | - [ ] Documentation about your PR is included in the `docs`, as needed. The docs live in a [separate repository](https://epi052.github.io/feroxbuster-docs/docs/). Update the appropriate pages at the links below. 20 | - [ ] update [example config file section](https://epi052.github.io/feroxbuster-docs/docs/configuration/ferox-config-toml/) 21 | - [ ] update [help output section](https://epi052.github.io/feroxbuster-docs/docs/configuration/command-line/) 22 | - [ ] add an [example](https://epi052.github.io/feroxbuster-docs/docs/examples/) 23 | 24 | ## Additional Tests 25 | - [ ] New code is unit tested 26 | - [ ] New code is integration tested, as needed 27 | - [ ] New tests pass 28 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 14 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | - confirmed 10 | # Label to use when marking an issue as stale 11 | staleLabel: stale 12 | # Comment to post when marking an issue as stale. Set to `false` to disable 13 | markComment: > 14 | This issue has been automatically marked as stale because it has not had 15 | recent activity. It will be closed if no further activity occurs. Thank you 16 | for your contributions. 17 | # Comment to post when closing a stale issue. Set to `false` to disable 18 | closeComment: false 19 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: CI Pipeline 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | check: 7 | name: Check 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - name: Cache cargo & target directories 12 | uses: Swatinem/rust-cache@v2 13 | - uses: dtolnay/rust-toolchain@stable 14 | - run: cargo check 15 | 16 | test: 17 | name: Test Suite 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Cache cargo & target directories 22 | uses: Swatinem/rust-cache@v2 23 | - name: Install latest nextest release 24 | uses: taiki-e/install-action@nextest 25 | - uses: dtolnay/rust-toolchain@stable 26 | - name: Test with latest nextest release 27 | run: cargo nextest run --all-features --all-targets --retries 4 --no-fail-fast 28 | 29 | fmt: 30 | name: Rust fmt 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v4 34 | - name: Cache cargo & target directories 35 | uses: Swatinem/rust-cache@v2 36 | - uses: dtolnay/rust-toolchain@stable 37 | - run: cargo fmt --all -- --check 38 | 39 | clippy: 40 | name: Clippy 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v4 44 | - name: Cache cargo & target directories 45 | uses: Swatinem/rust-cache@v2 46 | - uses: dtolnay/rust-toolchain@stable 47 | - run: cargo clippy --all-targets --all-features -- -D warnings 48 | -------------------------------------------------------------------------------- /.github/workflows/cicd-to-dockerhub.yml: -------------------------------------------------------------------------------- 1 | name: ci-to-dockerhub 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@v4 13 | 14 | - name: Login to Docker Hub 15 | uses: docker/login-action@v3 16 | with: 17 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 18 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 19 | 20 | - name: Set up Docker Buildx 21 | id: buildx 22 | uses: docker/setup-buildx-action@v3 23 | 24 | - name: Build and push 25 | id: docker_build 26 | uses: docker/build-push-action@v6 27 | with: 28 | context: ./ 29 | file: ./Dockerfile 30 | push: true 31 | tags: ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest 32 | 33 | - name: Image digest 34 | run: echo ${{ steps.docker_build.outputs.digest }} 35 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | on: [push] 2 | 3 | name: Code Coverage Pipeline 4 | 5 | jobs: 6 | coverage: 7 | name: LLVM Coverage 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - uses: dtolnay/rust-toolchain@stable 12 | with: 13 | components: llvm-tools-preview 14 | - name: Install cargo-llvm-cov and cargo-nextest 15 | uses: taiki-e/install-action@v2 16 | with: 17 | tool: cargo-nextest,cargo-llvm-cov 18 | - name: Generate code coverage 19 | run: cargo llvm-cov nextest --all-features --no-fail-fast --lcov --retries 4 --output-path lcov.info 20 | - name: Upload coverage to Codecov 21 | uses: codecov/codecov-action@v4 22 | with: 23 | files: lcov.info 24 | fail_ci_if_error: true 25 | token: ${{ secrets.CODECOV_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/winget.yml: -------------------------------------------------------------------------------- 1 | name: Publish to Winget 2 | on: 3 | release: 4 | types: [released] 5 | workflow_dispatch: 6 | inputs: 7 | tag_name: 8 | description: 'Tag name of release' 9 | required: true 10 | type: string 11 | 12 | jobs: 13 | publish: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: vedantmgoyal2009/winget-releaser@main 17 | with: 18 | identifier: epi052.feroxbuster 19 | installers-regex: '-windows-feroxbuster\.exe\.zip$' 20 | token: ${{ secrets.WINGET_TOKEN }} 21 | release-tag: ${{ inputs.tag_name || github.event.release.tag_name || github.ref_name }} 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # These are backup files generated by rustfmt 7 | **/*.rs.bk 8 | 9 | # jetbrains metadata folder 10 | .idea/ 11 | 12 | # vscode metadata folder 13 | .vscode/ 14 | 15 | # personal feroxbuster config for testing 16 | ferox-config.toml 17 | 18 | # images for the README on github 19 | img/** 20 | 21 | # scripts to check code coverage using nightly compiler 22 | check-coverage.sh 23 | lcov_cobertura.py 24 | 25 | # dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is 26 | .dockerignore 27 | 28 | # state file created during tests 29 | ferox-*.state 30 | 31 | # python stuff cuz reasons 32 | Pipfile* 33 | 34 | # ignore choco_package generated nupkg 35 | /choco_package/*.nupkg 36 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | reorder_modules = false 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "feroxbuster" 3 | version = "2.11.0" 4 | authors = ["Ben 'epi' Risher (@epi052)"] 5 | license = "MIT" 6 | edition = "2021" 7 | homepage = "https://github.com/epi052/feroxbuster" 8 | repository = "https://github.com/epi052/feroxbuster" 9 | description = "A fast, simple, recursive content discovery tool." 10 | categories = ["command-line-utilities"] 11 | keywords = [ 12 | "pentest", 13 | "enumeration", 14 | "url-bruteforce", 15 | "content-discovery", 16 | "web", 17 | ] 18 | exclude = [".github/*", "img/*", "check-coverage.sh"] 19 | build = "build.rs" 20 | 21 | [badges] 22 | maintenance = { status = "actively-developed" } 23 | 24 | [build-dependencies] 25 | clap = { version = "4.5", features = ["wrap_help", "cargo"] } 26 | clap_complete = "4.5" 27 | regex = "1.10" 28 | lazy_static = "1.5" 29 | dirs = "5.0" 30 | 31 | [dependencies] 32 | scraper = "0.19" 33 | futures = "0.3" 34 | tokio = { version = "1.39", features = ["full"] } 35 | tokio-util = { version = "0.7", features = ["codec"] } 36 | log = "0.4" 37 | env_logger = "0.11" 38 | reqwest = { version = "0.12", features = ["socks", "native-tls-alpn"] } 39 | # uses feature unification to add 'serde' to reqwest::Url 40 | url = { version = "2.5", features = ["serde"] } 41 | serde_regex = "1.1" 42 | clap = { version = "4.5", features = ["wrap_help", "cargo"] } 43 | lazy_static = "1.5" 44 | toml = "0.8" 45 | serde = { version = "1.0", features = ["derive", "rc"] } 46 | serde_json = "1.0" 47 | uuid = { version = "1.10", features = ["v4"] } 48 | indicatif = { version = "0.17.8" } 49 | console = "0.15" 50 | openssl = { version = "0.10", features = ["vendored"] } 51 | dirs = "5.0" 52 | regex = "1.10" 53 | crossterm = "0.27" 54 | rlimit = "0.10" 55 | ctrlc = "3.4" 56 | anyhow = "1.0" 57 | leaky-bucket = "1.1" 58 | gaoya = "0.2" 59 | # 0.37+ relies on the broken version of indicatif and forces 60 | # the broken version to be used regardless of the version 61 | # specified above 62 | self_update = { version = "0.40", features = [ 63 | "archive-tar", 64 | "compression-flate2", 65 | "archive-zip", 66 | "compression-zip-deflate", 67 | ] } 68 | 69 | [dev-dependencies] 70 | tempfile = "3.12" 71 | httpmock = "0.7" 72 | assert_cmd = "2.0" 73 | predicates = "3.1" 74 | 75 | [profile.release] 76 | lto = true 77 | codegen-units = 1 78 | panic = 'abort' 79 | 80 | [package.metadata.deb] 81 | section = "utility" 82 | license-file = ["LICENSE", "4"] 83 | conf-files = ["/etc/feroxbuster/ferox-config.toml"] 84 | assets = [ 85 | [ 86 | "target/release/feroxbuster", 87 | "/usr/bin/", 88 | "755", 89 | ], 90 | [ 91 | "ferox-config.toml.example", 92 | "/etc/feroxbuster/ferox-config.toml", 93 | "644", 94 | ], 95 | [ 96 | "shell_completions/feroxbuster.bash", 97 | "/usr/share/bash-completion/completions/feroxbuster.bash", 98 | "644", 99 | ], 100 | [ 101 | "shell_completions/feroxbuster.fish", 102 | "/usr/share/fish/completions/feroxbuster.fish", 103 | "644", 104 | ], 105 | [ 106 | "shell_completions/_feroxbuster", 107 | "/usr/share/zsh/vendor-completions/_feroxbuster", 108 | "644", 109 | ], 110 | ] 111 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.17.1 AS build 2 | LABEL maintainer="wfnintr@null.net" 3 | 4 | RUN apk upgrade --update-cache --available && apk add --update openssl 5 | 6 | # Download latest release 7 | RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip \ 8 | && unzip -d /tmp/ feroxbuster.zip feroxbuster \ 9 | && chmod +x /tmp/feroxbuster \ 10 | && wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-medium-directories.txt -O /tmp/raft-medium-directories.txt 11 | 12 | FROM alpine:3.17.1 AS release 13 | COPY --from=build /tmp/raft-medium-directories.txt /usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt 14 | COPY --from=build /tmp/feroxbuster /usr/local/bin/feroxbuster 15 | 16 | RUN adduser \ 17 | --gecos "" \ 18 | --disabled-password \ 19 | feroxbuster 20 | 21 | USER feroxbuster 22 | 23 | ENTRYPOINT ["feroxbuster"] 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-2023 epi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default_prefix = /usr/local 2 | prefix ?= $(default_prefix) 3 | exec_prefix = $(prefix) 4 | bindir = $(exec_prefix)/bin 5 | datarootdir = $(prefix)/share 6 | datadir = $(datarootdir) 7 | example_config = ferox-config.toml.example 8 | config_file = ferox-config.toml 9 | completion_dir = shell_completions 10 | completion_prefix = $(completion_dir)/$(BIN) 11 | 12 | BIN=feroxbuster 13 | SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock 14 | 15 | RELEASE = debug 16 | DEBUG ?= 0 17 | 18 | ifeq (0, $(DEBUG)) 19 | ARGS = --release 20 | RELEASE = release 21 | endif 22 | 23 | VENDORED ?= 0 24 | ifeq (1,$(VENDORED)) 25 | ARGS += --frozen 26 | endif 27 | 28 | TARGET = target/$(RELEASE) 29 | 30 | .PHONY: all clean install uninstall test update 31 | 32 | all: cli 33 | cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES) 34 | install: all install-cli 35 | 36 | verify: 37 | cargo fmt 38 | cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic 39 | cargo test 40 | 41 | clean: 42 | cargo clean 43 | 44 | vendor: vendor.tar 45 | 46 | vendor.tar: 47 | cargo vendor 48 | tar pcf vendor.tar vendor 49 | rm -rf vendor 50 | 51 | install-cli: cli 52 | install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash" 53 | install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish" 54 | install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)" 55 | install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)" 56 | install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz" 57 | install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)" 58 | 59 | uninstall: 60 | rm -f "$(DESTDIR)$(bindir)/$(BIN)" 61 | rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz" 62 | rm -rf "$(DESTDIR)/etc/$(BIN)/" 63 | rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash" 64 | rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)" 65 | rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish" 66 | 67 | extract: 68 | ifeq (1, $(VENDORED)) 69 | tar pxf vendor.tar 70 | endif 71 | 72 | $(TARGET)/$(BIN): extract 73 | mkdir -p .cargo debian 74 | touch debian/cargo.config 75 | cp debian/cargo.config .cargo/config.toml 76 | cargo build $(ARGS) 77 | 78 | $(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN) 79 | help2man --no-info $< | gzip -c > $@.partial 80 | mv $@.partial $@ 81 | -------------------------------------------------------------------------------- /Makefile.toml: -------------------------------------------------------------------------------- 1 | # composite tasks 2 | [tasks.upgrade] 3 | dependencies = ["upgrade-deps", "update"] 4 | 5 | [tasks.check] 6 | dependencies = ["fmt", "clippy", "test"] 7 | 8 | # cleaning 9 | [tasks.clean-state] 10 | script = """ 11 | rm ferox-*.state 12 | """ 13 | 14 | # dependency management 15 | [tasks.upgrade-deps] 16 | command = "cargo" 17 | args = ["upgrade", "--exclude", "self_update"] 18 | 19 | [tasks.update] 20 | command = "cargo" 21 | args = ["update"] 22 | 23 | # clippy / lint 24 | [tasks.clippy] 25 | clear = true 26 | script = """ 27 | cargo clippy --all-targets --all-features -- -D warnings 28 | """ 29 | 30 | [tasks.fmt] 31 | clear = true 32 | script = """ 33 | cargo fmt --all 34 | """ 35 | 36 | # tests 37 | [tasks.test] 38 | clear = true 39 | script = """ 40 | cargo nextest run --all-features --all-targets --retries 4 --no-fail-fast 41 | """ 42 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{copy, create_dir_all, OpenOptions}; 2 | use std::io::{Read, Seek, Write}; 3 | 4 | use clap_complete::{generate_to, shells}; 5 | 6 | include!("src/parser.rs"); 7 | 8 | fn main() { 9 | println!("cargo:rerun-if-env-changed=src/parser.rs"); 10 | 11 | if std::env::var("DOCS_RS").is_ok() { 12 | return; // only build when we're not generating docs 13 | } 14 | 15 | let outdir = "shell_completions"; 16 | 17 | let mut app = initialize(); 18 | 19 | generate_to(shells::Bash, &mut app, "feroxbuster", outdir).unwrap(); 20 | generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap(); 21 | generate_to(shells::Fish, &mut app, "feroxbuster", outdir).unwrap(); 22 | generate_to(shells::PowerShell, &mut app, "feroxbuster", outdir).unwrap(); 23 | generate_to(shells::Elvish, &mut app, "feroxbuster", outdir).unwrap(); 24 | 25 | // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we 26 | // landed on was to add -o plusdirs to the bash completion script. The following code aims to 27 | // automate that fix and have it present in all future builds 28 | let mut contents = String::new(); 29 | 30 | let mut bash_file = OpenOptions::new() 31 | .read(true) 32 | .write(true) 33 | .open(format!("{outdir}/feroxbuster.bash")) 34 | .expect("Couldn't open bash completion script"); 35 | 36 | bash_file 37 | .read_to_string(&mut contents) 38 | .expect("Couldn't read bash completion script"); 39 | 40 | contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster"); 41 | 42 | bash_file 43 | .rewind() 44 | .expect("Couldn't seek to position 0 in bash completion script"); 45 | 46 | bash_file 47 | .write_all(contents.as_bytes()) 48 | .expect("Couldn't write updated bash completion script to disk"); 49 | 50 | // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a 51 | // config file during the build process. The following code will place an example config in 52 | // the user's configuration directory 53 | // - linux: $XDG_CONFIG_HOME or $HOME/.config 54 | // - macOS: $HOME/Library/Application Support 55 | // - windows: {FOLDERID_RoamingAppData} 56 | let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory"); 57 | config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster 58 | 59 | if !config_dir.exists() { 60 | // recursively create the feroxbuster directory and all of its parent components if 61 | // they are missing 62 | if create_dir_all(&config_dir).is_err() { 63 | // only copy the config file when we're not running in the CI/CD pipeline 64 | // which fails with permission denied 65 | eprintln!("Couldn't create one or more directories needed to copy the config file"); 66 | return; 67 | } 68 | } 69 | 70 | // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME 71 | // ever changes, this will need to be updated 72 | let config_file = config_dir.join("ferox-config.toml"); 73 | 74 | if !config_file.exists() { 75 | // config file doesn't exist, add it to the config directory 76 | if copy("ferox-config.toml.example", config_file).is_err() { 77 | eprintln!("Couldn't copy example config into config directory"); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /choco_package/feroxbuster.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | feroxbuster 5 | 2.8.0 6 | https://github.com/epi052/feroxbuster/releases/ 7 | epi052 8 | feroxbuster (Install) 9 | epi052 10 | https://github.com/epi052/feroxbuster 11 | https://rawcdn.githack.com/epi052/feroxbuster/2d381e7e057ce60c580b324dd36c9abaf30c2ec7/img/logo/logo.png 12 | 2020-2023 13 | https://github.com/epi052/feroxbuster/blob/main/LICENSE 14 | true 15 | https://github.com/epi052/feroxbuster 16 | https://epi052.github.io/feroxbuster-docs/docs/ 17 | 18 | https://github.com/epi052/feroxbuster/issues 19 | content-discovery pentesting-tool url-bruteforcer 20 | A simple, fast, recursive content discovery tool written in Rust 21 | 22 | A simple, fast, recursive content discovery tool written in Rust 23 | [![Feroxbuster](https://github.com/epi052/feroxbuster/raw/main/img/logo/default-cropped.png)](https://github.com/epi052/feroxbuster) 24 | 25 | ## What the heck is a ferox anyway? 26 | 27 | Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a 28 | variation. 29 | 30 | ## What's it do tho? 31 | 32 | `feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing). 33 | 34 | Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web 35 | application, but are still accessible by an attacker. 36 | 37 | `feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These 38 | resources may store sensitive information about web applications and operational systems, such as source code, 39 | credentials, internal network addressing, etc... 40 | 41 | This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource 42 | Enumeration. 43 | 44 | ## Quick Start 45 | 46 | This section will cover the minimum amount of information to get up and running with feroxbuster. Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/), as it's much more comprehensive. 47 | 48 | ### Installation 49 | 50 | There are quite a few other [installation methods](https://epi052.github.io/feroxbuster-docs/docs/installation/), but these snippets should cover the majority of users. 51 | 52 | #### All others Docs 53 | 54 | Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/). 55 | 56 | ## Example Usage 57 | 58 | Here are a few brief examples to get you started. Please note, feroxbuster can do a **lot more** than what's listed below. As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/docs/). 59 | 60 | ### Multiple Values 61 | 62 | Options that take multiple values are very flexible. Consider the following ways of specifying extensions: 63 | 64 | ``` 65 | ./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx 66 | ``` 67 | 68 | The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url 69 | 70 | All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The 71 | same goes for urls, headers, status codes, queries, and size filters. 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /choco_package/legal/LICENSE.txt: -------------------------------------------------------------------------------- 1 |  2 | From: https://github.com/epi052/feroxbuster/blob/main/LICENSE 3 | 4 | LICENSE 5 | 6 | MIT License 7 | 8 | Copyright (c) 2020-2023 epi 9 | 10 | Permission is hereby granted, free of charge, to any person obtaining a copy 11 | of this software and associated documentation files (the "Software"), to deal 12 | in the Software without restriction, including without limitation the rights 13 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | copies of the Software, and to permit persons to whom the Software is 15 | furnished to do so, subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be included in all 18 | copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | SOFTWARE. 27 | -------------------------------------------------------------------------------- /choco_package/legal/VERIFICATION.txt: -------------------------------------------------------------------------------- 1 |  2 | VERIFICATION 3 | 4 | checksum -t sha512 -f .\x86-windows-feroxbuster.exe.zip 5 | checksum -t sha512 -f .\x86_64-windows-feroxbuster.exe.zip -------------------------------------------------------------------------------- /choco_package/tools/chocolateyinstall.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = 'Stop' 2 | 3 | $toolsDir = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)" 4 | $version = '2.8.0' 5 | $url = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86-windows-feroxbuster.exe.zip" 6 | $url64 = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86_64-windows-feroxbuster.exe.zip" 7 | 8 | $packageArgs = @{ 9 | packageName = $env:ChocolateyPackageName 10 | unzipLocation = $toolsDir 11 | fileType = 'exe' #only one of these: exe, msi, msu 12 | url = $url 13 | url64bit = $url64 14 | #file = $fileLocation 15 | 16 | softwareName = 'feroxbuster*' 17 | 18 | # Checksums are now required as of 0.10.0. 19 | # To determine checksums, you can get that from the original site if provided. 20 | # You can also use checksum.exe (choco install checksum) and use it 21 | # e.g. checksum -t sha256 -f path\to\file 22 | checksum = 'e5cac59c737260233903a17706a68bac11fe0d7a15169e1c5a9637cc221e7230fd6ddbfc1a7243833dde6472ad053c033449ca8338164654f7354363da54ba88' 23 | checksumType = 'sha512' 24 | checksum64 = 'cce58d6eacef7e12c31076f5a00fee9742a4e3fdfc69d807d98736200e50469f77359978e137ecafd87b14460845c65c6808d1f8b23ae561f7e7c637e355dee3' 25 | checksumType64= 'sha512' 26 | } 27 | Install-ChocolateyZipPackage @packageArgs # https://docs.chocolatey.org/en-us/create/functions/install-chocolateyzippackage -------------------------------------------------------------------------------- /choco_package/tools/chocolateyuninstall.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = 'Stop' # stop on all errors 2 | $packageArgs = @{ 3 | packageName = $env:ChocolateyPackageName 4 | softwareName = 'feroxbuster*' #part or all of the Display Name as you see it in Programs and Features. It should be enough to be unique 5 | fileType = 'exe' #only one of these: MSI or EXE (ignore MSU for now) 6 | } 7 | 8 | # Get-UninstallRegistryKey is new to 0.9.10, if supporting 0.9.9.x and below, 9 | # take a dependency on "chocolatey-core.extension" in your nuspec file. 10 | # This is only a fuzzy search if $softwareName includes '*'. Otherwise it is 11 | # exact. In the case of versions in key names, we recommend removing the version 12 | # and using '*'. 13 | [array]$key = Get-UninstallRegistryKey -SoftwareName $packageArgs['softwareName'] 14 | 15 | if ($key.Count -eq 1) { 16 | $key | % { 17 | $packageArgs['file'] = "$($_.UninstallString)" #NOTE: You may need to split this if it contains spaces, see below 18 | 19 | if ($packageArgs['fileType'] -eq 'MSI') { 20 | # The Product Code GUID is all that should be passed for MSI, and very 21 | # FIRST, because it comes directly after /x, which is already set in the 22 | # Uninstall-ChocolateyPackage msiargs (facepalm). 23 | $packageArgs['silentArgs'] = "$($_.PSChildName) $($packageArgs['silentArgs'])" 24 | 25 | # Don't pass anything for file, it is ignored for msi (facepalm number 2) 26 | # Alternatively if you need to pass a path to an msi, determine that and 27 | # use it instead of the above in silentArgs, still very first 28 | $packageArgs['file'] = '' 29 | } else { 30 | # NOTES: 31 | # - You probably will need to sanitize $packageArgs['file'] as it comes from the registry and could be in a variety of fun but unusable formats 32 | # - Split args from exe in $packageArgs['file'] and pass those args through $packageArgs['silentArgs'] or ignore them 33 | # - Ensure you don't pass double quotes in $file (aka $packageArgs['file']) - otherwise you will get "Illegal characters in path when you attempt to run this" 34 | # - Review the code for auto-uninstaller for all of the fun things it does in sanitizing - https://github.com/chocolatey/choco/blob/bfe351b7d10c798014efe4bfbb100b171db25099/src/chocolatey/infrastructure.app/services/AutomaticUninstallerService.cs#L142-L192 35 | } 36 | 37 | Uninstall-ChocolateyPackage @packageArgs 38 | } 39 | } elseif ($key.Count -eq 0) { 40 | Write-Warning "$packageName has already been uninstalled by other means." 41 | } elseif ($key.Count -gt 1) { 42 | Write-Warning "$($key.Count) matches found!" 43 | Write-Warning "To prevent accidental data loss, no programs will be uninstalled." 44 | Write-Warning "Please alert package maintainer the following keys were matched:" 45 | $key | % {Write-Warning "- $($_.DisplayName)"} 46 | } 47 | 48 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/docs/.nojekyll -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 |

The page has moved to: 8 | feroxbuster-docs

9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /ferox-config.toml.example: -------------------------------------------------------------------------------- 1 | # Example configuration for feroxbuster 2 | # 3 | # If you wish to provide persistent settings to feroxbuster, rename this file to ferox-config.toml and make sure 4 | # it resides in the same directory as the feroxbuster binary. 5 | # 6 | # After that, uncomment any line to override the default value provided by the binary itself. 7 | # 8 | # Any setting used here can be overridden by the corresponding command line option/argument 9 | # 10 | # wordlist = "/wordlists/seclists/Discovery/Web-Content/raft-medium-directories.txt" 11 | # status_codes = [200, 500] 12 | # filter_status = [301] 13 | # threads = 1 14 | # timeout = 5 15 | # proxy = "http://127.0.0.1:8080" 16 | # replay_proxy = "http://127.0.0.1:8081" 17 | # replay_codes = [200, 302] 18 | # verbosity = 1 19 | # parallel = 8 20 | # scan_limit = 6 21 | # rate_limit = 250 22 | # quiet = true 23 | # silent = true 24 | # auto_tune = true 25 | # auto_bail = true 26 | # json = true 27 | # output = "/targets/ellingson_mineral_company/gibson.txt" 28 | # debug_log = "/var/log/find-the-derp.log" 29 | # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0" 30 | # random_agent = false 31 | # redirects = true 32 | # insecure = true 33 | # collect_words = true 34 | # collect_backups = true 35 | # collect_extensions = true 36 | # extensions = ["php", "html"] 37 | # dont_collect = ["png", "gif", "jpg", "jpeg"] 38 | # methods = ["GET", "POST"] 39 | # data = [11, 12, 13, 14, 15] 40 | # url_denylist = ["http://dont-scan.me", "https://also-not.me"] 41 | # regex_denylist = ["/deny.*"] 42 | # no_recursion = true 43 | # add_slash = true 44 | # stdin = true 45 | # dont_filter = true 46 | # extract_links = true 47 | # depth = 1 48 | # limit_bars = 3 49 | # force_recursion = true 50 | # filter_size = [5174] 51 | # filter_regex = ["^ignore me$"] 52 | # filter_similar = ["https://somesite.com/soft404"] 53 | # filter_word_count = [993] 54 | # filter_line_count = [35, 36] 55 | # queries = [["name","value"], ["rick", "astley"]] 56 | # save_state = false 57 | # time_limit = "10m" 58 | # server_certs = ["/some/cert.pem", "/some/other/cert.pem"] 59 | # client_cert = "/some/client/cert.pem" 60 | # client_key = "/some/client/key.pem" 61 | # request_file = "/some/raw/request/file" 62 | # protocol = "http" 63 | # scan_dir_listings = true 64 | 65 | # headers can be specified on multiple lines or as an inline table 66 | # 67 | # inline example 68 | # headers = {"stuff" = "things"} 69 | # 70 | # multi-line example 71 | # note: if multi-line is used, all key/value pairs under it belong to the headers table until the next table 72 | # is found or the end of the file is reached 73 | # 74 | # If you want to use [headers], UNCOMMENT the line below 75 | # [headers] 76 | # stuff = "things" 77 | # more = "headers" 78 | -------------------------------------------------------------------------------- /img/auto-bail-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/auto-bail-demo.gif -------------------------------------------------------------------------------- /img/auto-tune-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/auto-tune-demo.gif -------------------------------------------------------------------------------- /img/cancel-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/cancel-menu.png -------------------------------------------------------------------------------- /img/cancel-scan.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/cancel-scan.gif -------------------------------------------------------------------------------- /img/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/demo.gif -------------------------------------------------------------------------------- /img/dir-scan-bar-explained.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/dir-scan-bar-explained.png -------------------------------------------------------------------------------- /img/extract-scan-cmp-normal.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/extract-scan-cmp-normal.gif -------------------------------------------------------------------------------- /img/insecure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/insecure.png -------------------------------------------------------------------------------- /img/limit-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/limit-demo.gif -------------------------------------------------------------------------------- /img/logo/default-cropped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/logo/default-cropped.png -------------------------------------------------------------------------------- /img/logo/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/logo/logo.png -------------------------------------------------------------------------------- /img/normal-scan-cmp-extract.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/normal-scan-cmp-extract.gif -------------------------------------------------------------------------------- /img/pause-resume-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/pause-resume-demo.gif -------------------------------------------------------------------------------- /img/rate-limit-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/rate-limit-demo.gif -------------------------------------------------------------------------------- /img/replay-proxy-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/replay-proxy-demo.gif -------------------------------------------------------------------------------- /img/response-bar-explained.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/response-bar-explained.png -------------------------------------------------------------------------------- /img/resumed-scan.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/resumed-scan.gif -------------------------------------------------------------------------------- /img/save-state.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/save-state.png -------------------------------------------------------------------------------- /img/small-term.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/small-term.png -------------------------------------------------------------------------------- /img/time-limit.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/time-limit.gif -------------------------------------------------------------------------------- /img/total-bar-explained.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/img/total-bar-explained.png -------------------------------------------------------------------------------- /install-nix.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download 4 | 5 | MAC_ZIP=x86_64-macos-feroxbuster.zip 6 | MAC_URL="$BASE_URL/$MAC_ZIP" 7 | 8 | LIN32_ZIP=x86-linux-feroxbuster.zip 9 | LIN32_URL="$BASE_URL/$LIN32_ZIP" 10 | 11 | LIN64_ZIP=x86_64-linux-feroxbuster.zip 12 | LIN64_URL="$BASE_URL/$LIN64_ZIP" 13 | 14 | EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf 15 | 16 | INSTALL_DIR="${1:-$(pwd)}" 17 | 18 | echo "[+] Installing feroxbuster to ${INSTALL_DIR}!" 19 | 20 | which unzip &>/dev/null 21 | if [ "$?" != "0" ]; then 22 | echo "[!] unzip not found, exiting. " 23 | exit -1 24 | fi 25 | 26 | if [[ "$(uname)" == "Darwin" ]]; then 27 | echo "[=] Found MacOS, downloading from $MAC_URL" 28 | 29 | curl -sLO "$MAC_URL" 30 | unzip -o "$MAC_ZIP" -d "${INSTALL_DIR}" >/dev/null 31 | rm "$MAC_ZIP" 32 | elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then 33 | if [[ $(getconf LONG_BIT) == 32 ]]; then 34 | echo "[=] Found 32-bit Linux, downloading from $LIN32_URL" 35 | 36 | curl -sLO "$LIN32_URL" 37 | unzip -o "$LIN32_ZIP" -d "${INSTALL_DIR}" >/dev/null 38 | rm "$LIN32_ZIP" 39 | else 40 | echo "[=] Found 64-bit Linux, downloading from $LIN64_URL" 41 | 42 | curl -sLO "$LIN64_URL" 43 | unzip -o "$LIN64_ZIP" -d "${INSTALL_DIR}" >/dev/null 44 | rm "$LIN64_ZIP" 45 | fi 46 | 47 | if [[ "$(fc-list NotoColorEmoji | wc -l)" -gt 0 ]]; then 48 | echo "[=] Found Noto Emoji Font, skipping install" 49 | else 50 | echo "[=] Installing Noto Emoji Font" 51 | mkdir -p ~/.fonts 52 | pushd ~/.fonts 2>&1 >/dev/null 53 | 54 | curl -sLO "$EMOJI_URL" 55 | 56 | fc-cache -f -v >/dev/null 57 | 58 | popd 2>&1 >/dev/null 59 | echo "[+] Noto Emoji Font installed" 60 | fi 61 | fi 62 | 63 | chmod +x "${INSTALL_DIR}/feroxbuster" 64 | 65 | echo "[+] Installed feroxbuster" 66 | echo " [-] path: ${INSTALL_DIR}/feroxbuster" 67 | echo " [-] version: $(${INSTALL_DIR}/feroxbuster -V | awk '{print $2}')" 68 | -------------------------------------------------------------------------------- /snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: feroxbuster 2 | version: git 3 | summary: A simple, fast, recursive content discovery tool written in Rust 4 | description: | 5 | feroxbuster is a tool designed to perform Forced Browsing. 6 | 7 | Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker. 8 | 9 | feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc... 10 | 11 | This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration. 12 | 13 | 14 | base: core18 15 | 16 | plugs: 17 | etc-feroxbuster: 18 | interface: system-files 19 | read: 20 | - /etc/feroxbuster 21 | dot-config-feroxbuster: 22 | interface: personal-files 23 | read: 24 | - $HOME/.config/feroxbuster 25 | 26 | architectures: 27 | - build-on: amd64 28 | - build-on: i386 29 | 30 | parts: 31 | feroxbuster: 32 | plugin: rust 33 | source: . 34 | 35 | apps: 36 | feroxbuster: 37 | command: bin/feroxbuster 38 | plugs: 39 | - etc-feroxbuster 40 | - dot-config-feroxbuster 41 | - network 42 | -------------------------------------------------------------------------------- /src/banner/entry.rs: -------------------------------------------------------------------------------- 1 | use console::{measure_text_width, Emoji}; 2 | use std::fmt; 3 | 4 | /// Initial visual indentation size used in formatting banner entries 5 | const INDENT: usize = 3; 6 | 7 | /// Column width used in formatting banner entries 8 | const COL_WIDTH: usize = 22; 9 | 10 | /// Represents a single line on the banner 11 | #[derive(Default)] 12 | pub(super) struct BannerEntry { 13 | /// emoji used in the banner entry 14 | emoji: String, 15 | 16 | /// title used in the banner entry 17 | title: String, 18 | 19 | /// value passed in via config/cli/defaults 20 | value: String, 21 | } 22 | 23 | /// implementation of a banner entry 24 | impl BannerEntry { 25 | /// Create a new banner entry from given fields 26 | pub fn new(emoji: &str, title: &str, value: &str) -> Self { 27 | BannerEntry { 28 | emoji: emoji.to_string(), 29 | title: title.to_string(), 30 | value: value.to_string(), 31 | } 32 | } 33 | 34 | /// Simple wrapper for emoji or fallback when terminal doesn't support emoji 35 | fn format_emoji(&self) -> String { 36 | let width = measure_text_width(&self.emoji); 37 | let pad_len = width * width; 38 | let pad = format!("{:) -> fmt::Result { 47 | write!( 48 | f, 49 | "\u{0020}{:\u{0020}( 13 | timeout: u64, 14 | user_agent: &str, 15 | redirects: bool, 16 | insecure: bool, 17 | headers: &HashMap, 18 | proxy: Option<&str>, 19 | server_certs: I, 20 | client_cert: Option<&str>, 21 | client_key: Option<&str>, 22 | ) -> Result 23 | where 24 | I: IntoIterator, 25 | I::Item: AsRef + std::fmt::Debug, 26 | { 27 | let policy = if redirects { 28 | Policy::limited(10) 29 | } else { 30 | Policy::none() 31 | }; 32 | 33 | let header_map: HeaderMap = headers.try_into()?; 34 | 35 | let mut client = Client::builder() 36 | .timeout(Duration::new(timeout, 0)) 37 | .user_agent(user_agent) 38 | .danger_accept_invalid_certs(insecure) 39 | .default_headers(header_map) 40 | .redirect(policy) 41 | .http1_title_case_headers(); 42 | 43 | if let Some(some_proxy) = proxy { 44 | if !some_proxy.is_empty() { 45 | // it's not an empty string; set the proxy 46 | let proxy_obj = Proxy::all(some_proxy)?; 47 | // just add the proxy to the client 48 | // don't build and return it just yet 49 | client = client.proxy(proxy_obj); 50 | } 51 | } 52 | 53 | for cert_path in server_certs { 54 | let buf = std::fs::read(&cert_path)?; 55 | 56 | let cert = match reqwest::Certificate::from_pem(&buf) { 57 | Ok(cert) => cert, 58 | Err(err) => reqwest::Certificate::from_der(&buf).with_context(|| { 59 | format!( 60 | "{:?} does not contain a valid PEM or DER certificate\n{}", 61 | &cert_path, err 62 | ) 63 | })?, 64 | }; 65 | 66 | client = client.add_root_certificate(cert); 67 | } 68 | 69 | if let (Some(cert_path), Some(key_path)) = (client_cert, client_key) { 70 | if !cert_path.is_empty() && !key_path.is_empty() { 71 | let cert = std::fs::read(cert_path)?; 72 | let key = std::fs::read(key_path)?; 73 | 74 | let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| { 75 | format!( 76 | "either {} or {} are invalid; expecting PEM encoded certificate and key", 77 | cert_path, key_path 78 | ) 79 | })?; 80 | 81 | client = client.identity(identity); 82 | } 83 | } 84 | 85 | Ok(client.build()?) 86 | } 87 | 88 | #[cfg(test)] 89 | mod tests { 90 | use super::*; 91 | 92 | #[test] 93 | #[should_panic] 94 | /// create client with a bad proxy, expect panic 95 | fn client_with_bad_proxy() { 96 | let headers = HashMap::new(); 97 | initialize( 98 | 0, 99 | "stuff", 100 | true, 101 | false, 102 | &headers, 103 | Some("not a valid proxy"), 104 | Vec::::new(), 105 | None, 106 | None, 107 | ) 108 | .unwrap(); 109 | } 110 | 111 | #[test] 112 | /// create client with a proxy, expect no error 113 | fn client_with_good_proxy() { 114 | let headers = HashMap::new(); 115 | let proxy = "http://127.0.0.1:8080"; 116 | initialize( 117 | 0, 118 | "stuff", 119 | true, 120 | true, 121 | &headers, 122 | Some(proxy), 123 | Vec::::new(), 124 | None, 125 | None, 126 | ) 127 | .unwrap(); 128 | } 129 | 130 | #[test] 131 | /// create client with a server cert in pem format, expect no error 132 | fn client_with_valid_server_pem() { 133 | let headers = HashMap::new(); 134 | 135 | initialize( 136 | 0, 137 | "stuff", 138 | true, 139 | true, 140 | &headers, 141 | None, 142 | vec!["tests/mutual-auth/certs/server/server.crt.1".to_string()], 143 | None, 144 | None, 145 | ) 146 | .unwrap(); 147 | } 148 | 149 | #[test] 150 | /// create client with a server cert in der format, expect no error 151 | fn client_with_valid_server_der() { 152 | let headers = HashMap::new(); 153 | 154 | initialize( 155 | 0, 156 | "stuff", 157 | true, 158 | true, 159 | &headers, 160 | None, 161 | vec!["tests/mutual-auth/certs/server/server.der".to_string()], 162 | None, 163 | None, 164 | ) 165 | .unwrap(); 166 | } 167 | 168 | #[test] 169 | /// create client with two server certs (pem and der), expect no error 170 | fn client_with_valid_server_pem_and_der() { 171 | let headers = HashMap::new(); 172 | 173 | println!("{}", std::env::current_dir().unwrap().display()); 174 | 175 | initialize( 176 | 0, 177 | "stuff", 178 | true, 179 | true, 180 | &headers, 181 | None, 182 | vec![ 183 | "tests/mutual-auth/certs/server/server.crt.1".to_string(), 184 | "tests/mutual-auth/certs/server/server.der".to_string(), 185 | ], 186 | None, 187 | None, 188 | ) 189 | .unwrap(); 190 | } 191 | 192 | /// create client with invalid certificate, expect panic 193 | #[test] 194 | #[should_panic] 195 | fn client_with_invalid_server_cert() { 196 | let headers = HashMap::new(); 197 | 198 | initialize( 199 | 0, 200 | "stuff", 201 | true, 202 | true, 203 | &headers, 204 | None, 205 | vec!["tests/mutual-auth/certs/client/client.key".to_string()], 206 | None, 207 | None, 208 | ) 209 | .unwrap(); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/config/mod.rs: -------------------------------------------------------------------------------- 1 | //! all logic related to instantiating a running configuration 2 | 3 | mod container; 4 | mod utils; 5 | #[cfg(test)] 6 | mod tests; 7 | 8 | pub use self::container::Configuration; 9 | pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy}; 10 | -------------------------------------------------------------------------------- /src/event_handlers/command.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use reqwest::StatusCode; 5 | use tokio::sync::oneshot::Sender; 6 | 7 | use crate::response::FeroxResponse; 8 | use crate::{ 9 | event_handlers::Handles, 10 | message::FeroxMessage, 11 | statistics::{StatError, StatField}, 12 | traits::FeroxFilter, 13 | }; 14 | 15 | /// Protocol definition for updating an event handler via mpsc 16 | #[derive(Debug)] 17 | pub enum Command { 18 | /// Add one to the total number of requests 19 | AddRequest, 20 | 21 | /// Add one to the proper field(s) based on the given `StatError` 22 | AddError(StatError), 23 | 24 | /// Add one to the proper field(s) based on the given `StatusCode` 25 | AddStatus(StatusCode), 26 | 27 | /// Create the progress bar (`BarType::Total`) that is updated from the stats thread 28 | /// 29 | /// the u64 value is the offset at which to start the progress bar (can be 0) 30 | CreateBar(u64), 31 | 32 | /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value 33 | AddToUsizeField(StatField, usize), 34 | 35 | /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value 36 | SubtractFromUsizeField(StatField, usize), 37 | 38 | /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value 39 | AddToF64Field(StatField, f64), 40 | 41 | /// Save a `Stats` object to disk using `reporter::get_cached_file_handle` 42 | Save, 43 | 44 | /// Load a `Stats` object from disk 45 | LoadStats(String), 46 | 47 | /// Add a `FeroxFilter` implementor to `FilterHandler`'s instance of `FeroxFilters` 48 | AddFilter(Box), 49 | 50 | /// Remove a set of `FeroxFilter` implementors from `FeroxFilters` by index 51 | RemoveFilters(Vec), 52 | 53 | /// Send a `FeroxResponse` to the output handler for reporting 54 | Report(Box), 55 | 56 | /// Send a group of urls to be scanned (only used for the urls passed in explicitly by the user) 57 | ScanInitialUrls(Vec), 58 | 59 | /// Send a single url to be scanned (presumably added from the interactive menu) 60 | ScanNewUrl(String), 61 | 62 | /// Determine whether or not recursion is appropriate, given a FeroxResponse, if so start a scan 63 | TryRecursion(Box), 64 | 65 | /// Send a pointer to the wordlist to the recursion handler 66 | UpdateWordlist(Arc>), 67 | 68 | /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done 69 | JoinTasks(Sender), 70 | 71 | /// Command used to test that a spawned task succeeded in initialization 72 | Ping, 73 | 74 | /// Just receive a sender and reply, used for slowing down the main thread 75 | Sync(Sender), 76 | 77 | /// Notify event handler that a new extension has been seen 78 | AddDiscoveredExtension(String), 79 | 80 | /// Write an arbitrary string to disk 81 | WriteToDisk(Box), 82 | 83 | /// Break out of the (infinite) mpsc receive loop 84 | Exit, 85 | 86 | /// Give a handler access to an Arc instance after the handler has 87 | /// already been initialized 88 | AddHandles(Arc), 89 | 90 | /// inform the Stats object about which targets are being scanned 91 | UpdateTargets(Vec), 92 | 93 | /// query the Stats handler about the position of the overall progress bar 94 | QueryOverallBarEta(Sender), 95 | } 96 | -------------------------------------------------------------------------------- /src/event_handlers/container.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::config::Configuration; 3 | use crate::event_handlers::scans::ScanHandle; 4 | use crate::scan_manager::FeroxScans; 5 | use crate::Joiner; 6 | #[cfg(test)] 7 | use crate::{filters::FeroxFilters, statistics::Stats, Command}; 8 | use anyhow::{bail, Result}; 9 | use std::collections::HashSet; 10 | use std::sync::{Arc, RwLock}; 11 | #[cfg(test)] 12 | use tokio::sync::mpsc::{self, UnboundedReceiver}; 13 | 14 | #[derive(Debug)] 15 | /// Simple container for multiple JoinHandles 16 | pub struct Tasks { 17 | /// JoinHandle for terminal handler 18 | pub terminal: Joiner, 19 | 20 | /// JoinHandle for statistics handler 21 | pub stats: Joiner, 22 | 23 | /// JoinHandle for filters handler 24 | pub filters: Joiner, 25 | 26 | /// JoinHandle for scans handler 27 | pub scans: Joiner, 28 | } 29 | 30 | /// Tasks implementation 31 | impl Tasks { 32 | /// Given JoinHandles for terminal, statistics, and filters create a new Tasks object 33 | pub fn new(terminal: Joiner, stats: Joiner, filters: Joiner, scans: Joiner) -> Self { 34 | Self { 35 | terminal, 36 | stats, 37 | filters, 38 | scans, 39 | } 40 | } 41 | } 42 | 43 | #[derive(Debug)] 44 | /// Container for the different *Handles that will be shared across modules 45 | pub struct Handles { 46 | /// Handle for statistics 47 | pub stats: StatsHandle, 48 | 49 | /// Handle for filters 50 | pub filters: FiltersHandle, 51 | 52 | /// Handle for output (terminal/file) 53 | pub output: TermOutHandle, 54 | 55 | /// Handle for Configuration 56 | pub config: Arc, 57 | 58 | /// Handle for recursion 59 | pub scans: RwLock>, 60 | 61 | /// Pointer to the list of words generated from reading in the wordlist 62 | pub wordlist: Arc>, 63 | } 64 | 65 | /// implementation of Handles 66 | impl Handles { 67 | /// Given a StatsHandle, FiltersHandle, and OutputHandle, create a Handles object 68 | pub fn new( 69 | stats: StatsHandle, 70 | filters: FiltersHandle, 71 | output: TermOutHandle, 72 | config: Arc, 73 | wordlist: Arc>, 74 | ) -> Self { 75 | Self { 76 | stats, 77 | filters, 78 | output, 79 | config, 80 | scans: RwLock::new(None), 81 | wordlist, 82 | } 83 | } 84 | 85 | /// create a Handles object suitable for unit testing (non-functional) 86 | #[cfg(test)] 87 | pub fn for_testing( 88 | scanned_urls: Option>, 89 | config: Option>, 90 | ) -> (Self, UnboundedReceiver) { 91 | let configuration = config.unwrap_or_else(|| Arc::new(Configuration::new().unwrap())); 92 | let (tx, rx) = mpsc::unbounded_channel::(); 93 | let terminal_handle = TermOutHandle::new(tx.clone(), tx.clone()); 94 | let stats_handle = StatsHandle::new(Arc::new(Stats::new(configuration.json)), tx.clone()); 95 | let filters_handle = FiltersHandle::new(Arc::new(FeroxFilters::default()), tx.clone()); 96 | let wordlist = Arc::new(vec![String::from("this_is_a_test")]); 97 | let handles = Self::new( 98 | stats_handle, 99 | filters_handle, 100 | terminal_handle, 101 | configuration, 102 | wordlist, 103 | ); 104 | if let Some(sh) = scanned_urls { 105 | let scan_handle = ScanHandle::new(sh, tx); 106 | handles.set_scan_handle(scan_handle); 107 | } 108 | (handles, rx) 109 | } 110 | 111 | /// Set the ScanHandle object 112 | pub fn set_scan_handle(&self, handle: ScanHandle) { 113 | if let Ok(mut guard) = self.scans.write() { 114 | if guard.is_none() { 115 | let _ = std::mem::replace(&mut *guard, Some(handle)); 116 | } 117 | } 118 | } 119 | 120 | /// Helper to easily send a Command over the (locked) underlying CommandSender object 121 | pub fn send_scan_command(&self, command: Command) -> Result<()> { 122 | if let Ok(guard) = self.scans.read().as_ref() { 123 | if let Some(handle) = guard.as_ref() { 124 | handle.send(command)?; 125 | return Ok(()); 126 | } 127 | } 128 | 129 | bail!("Could not get underlying CommandSender object") 130 | } 131 | 132 | /// wrapper to reach into `FeroxScans` and yank out the length of `collected_extensions` 133 | pub fn num_collected_extensions(&self) -> usize { 134 | if !self.config.collect_extensions { 135 | // if --collect-extensions wasn't used, simply return 0 and forego unlocking 136 | return 0; 137 | } 138 | 139 | self.collected_extensions().len() 140 | } 141 | 142 | /// wrapper to reach into `FeroxScans` and yank out the length of `collected_extensions` 143 | pub fn collected_extensions(&self) -> HashSet { 144 | if let Ok(scans) = self.ferox_scans() { 145 | if let Ok(extensions) = scans.collected_extensions.read() { 146 | return extensions.clone(); 147 | } 148 | } 149 | 150 | HashSet::new() 151 | } 152 | 153 | /// number of words in the wordlist, multiplied by `expected_num_requests_multiplier` 154 | pub fn expected_num_requests_per_dir(&self) -> usize { 155 | let num_words = self.wordlist.len(); 156 | let multiplier = self.expected_num_requests_multiplier(); 157 | multiplier * num_words 158 | } 159 | 160 | /// number of extensions plus the number of request method types plus any dynamically collected 161 | /// extensions 162 | pub fn expected_num_requests_multiplier(&self) -> usize { 163 | let mut multiplier = self.config.extensions.len().max(1); 164 | 165 | if multiplier > 1 { 166 | // when we have more than one extension, we need to account for the fact that we'll 167 | // be making a request for each extension and the base word (e.g. /foo.html and /foo) 168 | multiplier += 1; 169 | } 170 | 171 | multiplier *= self.config.methods.len().max(1) * self.num_collected_extensions().max(1); 172 | 173 | multiplier 174 | } 175 | 176 | /// Helper to easily get the (locked) underlying FeroxScans object 177 | pub fn ferox_scans(&self) -> Result> { 178 | if let Ok(guard) = self.scans.read().as_ref() { 179 | if let Some(handle) = guard.as_ref() { 180 | return Ok(handle.data.clone()); 181 | } 182 | } 183 | 184 | bail!("Could not get underlying FeroxScans") 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/event_handlers/filters.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::filters::EmptyFilter; 3 | use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner}; 4 | use anyhow::Result; 5 | use std::sync::Arc; 6 | use tokio::sync::{ 7 | mpsc::{self, UnboundedReceiver}, 8 | oneshot, 9 | }; 10 | 11 | #[derive(Debug)] 12 | /// Container for filters transmitter and FeroxFilters object 13 | pub struct FiltersHandle { 14 | /// FeroxFilters object used across modules to track active filters 15 | pub data: Arc, 16 | 17 | /// transmitter used to update `data` 18 | pub tx: CommandSender, 19 | } 20 | 21 | /// implementation of FiltersHandle 22 | impl FiltersHandle { 23 | /// Given an Arc-wrapped FeroxFilters and CommandSender, create a new FiltersHandle 24 | pub fn new(data: Arc, tx: CommandSender) -> Self { 25 | Self { data, tx } 26 | } 27 | 28 | /// Send the given Command over `tx` 29 | pub fn send(&self, command: Command) -> Result<()> { 30 | self.tx.send(command)?; 31 | Ok(()) 32 | } 33 | 34 | /// Sync the handle with the handler 35 | pub async fn sync(&self) -> Result<()> { 36 | let (tx, rx) = oneshot::channel::(); 37 | self.send(Command::Sync(tx))?; 38 | rx.await?; 39 | Ok(()) 40 | } 41 | } 42 | 43 | /// event handler for updating a single data structure of all active filters 44 | #[derive(Debug)] 45 | pub struct FiltersHandler { 46 | /// collection of FeroxFilters 47 | data: Arc, 48 | 49 | /// Receiver half of mpsc from which `Command`s are processed 50 | receiver: UnboundedReceiver, 51 | } 52 | 53 | /// implementation of event handler for filters 54 | impl FiltersHandler { 55 | /// create new event handler 56 | pub fn new(data: Arc, receiver: UnboundedReceiver) -> Self { 57 | Self { data, receiver } 58 | } 59 | 60 | /// Initialize new `FeroxFilters` and the sc side of an mpsc channel that is responsible for 61 | /// updates to the aforementioned object. 62 | pub fn initialize() -> (Joiner, FiltersHandle) { 63 | log::trace!("enter: initialize"); 64 | 65 | let data = Arc::new(FeroxFilters::default()); 66 | let (tx, rx): FeroxChannel = mpsc::unbounded_channel(); 67 | 68 | let mut handler = Self::new(data.clone(), rx); 69 | 70 | let task = tokio::spawn(async move { handler.start().await }); 71 | 72 | let event_handle = FiltersHandle::new(data, tx); 73 | 74 | log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle); 75 | 76 | (task, event_handle) 77 | } 78 | 79 | /// Start a single consumer task (sc side of mpsc) 80 | /// 81 | /// The consumer simply receives `Command` and acts accordingly 82 | pub async fn start(&mut self) -> Result<()> { 83 | log::trace!("enter: start({:?})", self); 84 | 85 | while let Some(command) = self.receiver.recv().await { 86 | match command { 87 | Command::AddFilter(filter) => { 88 | if filter.as_any().downcast_ref::().is_none() { 89 | // don't add an empty filter 90 | self.data.push(filter)?; 91 | } 92 | } 93 | Command::RemoveFilters(mut indices) => self.data.remove(&mut indices), 94 | Command::Sync(sender) => { 95 | log::debug!("filters: {:?}", self); 96 | sender.send(true).unwrap_or_default(); 97 | } 98 | Command::Exit => break, 99 | _ => {} // no other commands needed for FilterHandler 100 | } 101 | } 102 | 103 | log::trace!("exit: start"); 104 | Ok(()) 105 | } 106 | } 107 | 108 | #[cfg(test)] 109 | mod tests { 110 | use super::*; 111 | use crate::filters::WordsFilter; 112 | 113 | #[tokio::test(flavor = "multi_thread", worker_threads = 1)] 114 | async fn empty_filter_skipped() { 115 | let data = Arc::new(FeroxFilters::default()); 116 | let (tx, rx): FeroxChannel = mpsc::unbounded_channel(); 117 | 118 | let mut handler = FiltersHandler::new(data.clone(), rx); 119 | 120 | let event_handle = FiltersHandle::new(data, tx); 121 | 122 | let _task = tokio::spawn(async move { handler.start().await }); 123 | 124 | event_handle 125 | .send(Command::AddFilter(Box::new(EmptyFilter {}))) 126 | .unwrap(); 127 | 128 | let (tx, rx) = oneshot::channel::(); 129 | event_handle.send(Command::Sync(tx)).unwrap(); 130 | rx.await.unwrap(); 131 | 132 | assert!(event_handle.data.filters.read().unwrap().is_empty()); 133 | 134 | event_handle 135 | .send(Command::AddFilter(Box::new(WordsFilter { word_count: 1 }))) 136 | .unwrap(); 137 | 138 | let (tx, rx) = oneshot::channel::(); 139 | event_handle.send(Command::Sync(tx)).unwrap(); 140 | rx.await.unwrap(); 141 | 142 | assert_eq!(event_handle.data.filters.read().unwrap().len(), 1); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/event_handlers/inputs.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::{ 3 | progress::PROGRESS_PRINTER, 4 | scan_manager::{FeroxState, PAUSE_SCAN}, 5 | scanner::RESPONSES, 6 | statistics::StatError, 7 | utils::slugify_filename, 8 | utils::{open_file, write_to}, 9 | SLEEP_DURATION, 10 | }; 11 | use anyhow::Result; 12 | use console::style; 13 | use crossterm::event::{self, Event, KeyCode}; 14 | use std::{ 15 | env::temp_dir, 16 | sync::{ 17 | atomic::{AtomicBool, Ordering}, 18 | Arc, 19 | }, 20 | thread::sleep, 21 | time::Duration, 22 | }; 23 | 24 | /// Atomic boolean flag, used to determine whether or not the terminal input handler should exit 25 | pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false); 26 | 27 | /// Container for filters transmitter and FeroxFilters object 28 | pub struct TermInputHandler { 29 | /// handles to other handlers 30 | handles: Arc, 31 | } 32 | 33 | /// implementation of event handler for terminal input 34 | /// 35 | /// kicks off the following handlers related to terminal input: 36 | /// ctrl+c handler that saves scan state to disk 37 | /// enter handler that listens for enter during scans to drop into interactive scan management menu 38 | impl TermInputHandler { 39 | /// Create new event handler 40 | pub fn new(handles: Arc) -> Self { 41 | Self { handles } 42 | } 43 | 44 | /// Initialize the sigint and enter handlers that are responsible for handling initial user 45 | /// interaction during scans 46 | pub fn initialize(handles: Arc) { 47 | log::trace!("enter: initialize({:?})", handles); 48 | 49 | let handler = Self::new(handles); 50 | handler.start(); 51 | 52 | log::trace!("exit: initialize"); 53 | } 54 | 55 | /// wrapper around sigint_handler and enter_handler 56 | fn start(&self) { 57 | tokio::task::spawn_blocking(Self::enter_handler); 58 | 59 | if self.handles.config.save_state { 60 | // start the ctrl+c handler 61 | let cloned = self.handles.clone(); 62 | 63 | let result = ctrlc::set_handler(move || { 64 | let _ = Self::sigint_handler(cloned.clone()); 65 | }); 66 | 67 | if result.is_err() { 68 | log::warn!("Could not set Ctrl+c handler; scan state will not be saved"); 69 | self.handles 70 | .stats 71 | .send(Command::AddError(StatError::Other)) 72 | .unwrap_or_default(); 73 | } 74 | } 75 | } 76 | 77 | /// Writes the current state of the program to disk (if save_state is true) and then exits 78 | pub fn sigint_handler(handles: Arc) -> Result<()> { 79 | log::trace!("enter: sigint_handler({:?})", handles); 80 | 81 | let filename = if !handles.config.target_url.is_empty() { 82 | // target url populated 83 | slugify_filename(&handles.config.target_url, "ferox", "state") 84 | } else { 85 | // stdin used 86 | slugify_filename("stdin", "ferox", "state") 87 | }; 88 | 89 | let warning = format!( 90 | "🚨 Caught {} 🚨 saving scan state to {} ...", 91 | style("ctrl+c").yellow(), 92 | filename 93 | ); 94 | 95 | PROGRESS_PRINTER.println(warning); 96 | 97 | let state = FeroxState::new( 98 | handles.ferox_scans()?, 99 | handles.config.clone(), 100 | &RESPONSES, 101 | handles.stats.data.clone(), 102 | handles.filters.data.clone(), 103 | ); 104 | 105 | // User didn't set the --no-state flag (so saved_state is still the default true) 106 | if handles.config.save_state { 107 | let Ok(mut state_file) = open_file(&filename) else { 108 | // couldn't open the file, let the user know we're going to try again 109 | let error = format!( 110 | "❌ Could not save {}, falling back to {}", 111 | filename, 112 | temp_dir().to_string_lossy() 113 | ); 114 | PROGRESS_PRINTER.println(error); 115 | 116 | let temp_filename = temp_dir().join(&filename); 117 | 118 | let Ok(mut state_file) = open_file(&temp_filename.to_string_lossy()) else { 119 | // couldn't open the fallback file, let the user know 120 | let error = format!("❌❌ Could not save {:?}, giving up...", temp_filename); 121 | PROGRESS_PRINTER.println(error); 122 | 123 | log::trace!("exit: sigint_handler (failed to write)"); 124 | std::process::exit(1); 125 | }; 126 | 127 | write_to(&state, &mut state_file, true)?; 128 | 129 | let msg = format!("✅ Saved scan state to {:?}", temp_filename); 130 | PROGRESS_PRINTER.println(msg); 131 | 132 | log::trace!("exit: sigint_handler (saved to temp folder)"); 133 | std::process::exit(1); 134 | }; 135 | 136 | write_to(&state, &mut state_file, true)?; 137 | } 138 | 139 | log::trace!("exit: sigint_handler (end of program)"); 140 | std::process::exit(1); 141 | } 142 | 143 | /// Handles specific key events triggered by the user over stdin 144 | fn enter_handler() { 145 | // todo eventually move away from atomics, the blocking recv is the problem 146 | log::trace!("enter: start_enter_handler"); 147 | 148 | loop { 149 | if PAUSE_SCAN.load(Ordering::Relaxed) { 150 | // if the scan is already paused, we don't want this event poller fighting the user 151 | // over stdin 152 | sleep(Duration::from_millis(SLEEP_DURATION)); 153 | } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) { 154 | // It's guaranteed that the `read()` won't block when the `poll()` 155 | // function returns `true` 156 | 157 | if let Ok(key_pressed) = event::read() { 158 | // ignore any other keys 159 | if key_pressed == Event::Key(KeyCode::Enter.into()) { 160 | // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu 161 | // will be triggered and will handle setting PAUSE_SCAN to false 162 | PAUSE_SCAN.store(true, Ordering::Release); 163 | } 164 | } 165 | } else { 166 | // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE 167 | if SCAN_COMPLETE.load(Ordering::Relaxed) { 168 | // scan has been marked complete by main, time to exit the loop 169 | break; 170 | } 171 | } 172 | } 173 | log::trace!("exit: start_enter_handler"); 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/event_handlers/mod.rs: -------------------------------------------------------------------------------- 1 | //! collection of event handlers (typically long-running tokio spawned tasks) 2 | mod statistics; 3 | mod filters; 4 | mod container; 5 | mod command; 6 | mod outputs; 7 | mod scans; 8 | mod inputs; 9 | 10 | pub use self::command::Command; 11 | pub use self::container::{Handles, Tasks}; 12 | pub use self::filters::{FiltersHandle, FiltersHandler}; 13 | pub use self::inputs::{TermInputHandler, SCAN_COMPLETE}; 14 | pub use self::outputs::{TermOutHandle, TermOutHandler}; 15 | pub use self::scans::{ScanHandle, ScanHandler}; 16 | pub use self::statistics::{StatsHandle, StatsHandler}; 17 | -------------------------------------------------------------------------------- /src/event_handlers/statistics.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::{ 3 | config::Configuration, 4 | progress::{add_bar, BarType}, 5 | statistics::{StatField, Stats}, 6 | CommandSender, FeroxChannel, Joiner, 7 | }; 8 | use anyhow::Result; 9 | use console::style; 10 | use indicatif::ProgressBar; 11 | use std::{sync::Arc, time::Instant}; 12 | use tokio::sync::{ 13 | mpsc::{self, UnboundedReceiver}, 14 | oneshot, 15 | }; 16 | 17 | #[derive(Debug)] 18 | /// Container for statistics transmitter and Stats object 19 | pub struct StatsHandle { 20 | /// Stats object used across modules to track statistics 21 | pub data: Arc, 22 | 23 | /// transmitter used to update `data` 24 | pub tx: CommandSender, 25 | } 26 | 27 | /// implementation of StatsHandle 28 | impl StatsHandle { 29 | /// Given an Arc-wrapped Stats and CommandSender, create a new StatsHandle 30 | pub fn new(data: Arc, tx: CommandSender) -> Self { 31 | Self { data, tx } 32 | } 33 | 34 | /// Send the given Command over `tx` 35 | pub fn send(&self, command: Command) -> Result<()> { 36 | self.tx.send(command)?; 37 | Ok(()) 38 | } 39 | 40 | /// Sync the handle with the handler 41 | pub async fn sync(&self) -> Result<()> { 42 | let (tx, rx) = oneshot::channel::(); 43 | self.send(Command::Sync(tx))?; 44 | rx.await?; 45 | Ok(()) 46 | } 47 | } 48 | 49 | /// event handler struct for updating statistics 50 | #[derive(Debug)] 51 | pub struct StatsHandler { 52 | /// overall scan's progress bar 53 | bar: ProgressBar, 54 | 55 | /// Receiver half of mpsc from which `StatCommand`s are processed 56 | receiver: UnboundedReceiver, 57 | 58 | /// data class that stores all statistics updates 59 | stats: Arc, 60 | } 61 | 62 | /// implementation of event handler for statistics 63 | impl StatsHandler { 64 | /// create new event handler 65 | fn new(stats: Arc, rx_stats: UnboundedReceiver) -> Self { 66 | // will be updated later via StatCommand; delay is for banner to print first 67 | let bar = ProgressBar::hidden(); 68 | 69 | Self { 70 | bar, 71 | stats, 72 | receiver: rx_stats, 73 | } 74 | } 75 | 76 | /// Start a single consumer task (sc side of mpsc) 77 | /// 78 | /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate 79 | async fn start(&mut self, output_file: &str) -> Result<()> { 80 | log::trace!("enter: start({:?})", self); 81 | 82 | let start = Instant::now(); 83 | 84 | while let Some(command) = self.receiver.recv().await { 85 | match command as Command { 86 | Command::AddError(err) => { 87 | self.stats.add_error(err); 88 | self.increment_bar(); 89 | } 90 | Command::AddStatus(status) => { 91 | self.stats.add_status_code(status); 92 | 93 | self.increment_bar(); 94 | } 95 | Command::AddRequest => { 96 | self.stats.add_request(); 97 | self.increment_bar(); 98 | } 99 | Command::Save => { 100 | self.stats 101 | .save(start.elapsed().as_secs_f64(), output_file)?; 102 | } 103 | Command::AddToUsizeField(field, value) => { 104 | self.stats.update_usize_field(field, value); 105 | 106 | if matches!(field, StatField::TotalScans | StatField::TotalExpected) { 107 | self.bar.set_length(self.stats.total_expected() as u64); 108 | } 109 | } 110 | Command::SubtractFromUsizeField(field, value) => { 111 | self.stats.subtract_from_usize_field(field, value); 112 | 113 | if matches!(field, StatField::TotalExpected) { 114 | self.bar.set_length(self.stats.total_expected() as u64); 115 | } 116 | } 117 | Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value), 118 | Command::CreateBar(offset) => { 119 | self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total); 120 | self.bar.set_position(offset); 121 | } 122 | Command::LoadStats(filename) => { 123 | self.stats.merge_from(&filename)?; 124 | } 125 | Command::Sync(sender) => { 126 | sender.send(true).unwrap_or_default(); 127 | } 128 | Command::QueryOverallBarEta(sender) => { 129 | sender.send(self.bar.eta()).unwrap_or_default(); 130 | } 131 | Command::UpdateTargets(targets) => { 132 | self.stats.update_targets(targets); 133 | } 134 | Command::Exit => break, 135 | _ => {} // no more commands needed 136 | } 137 | } 138 | 139 | self.bar.finish(); 140 | 141 | log::info!("{:#?}", *self.stats); 142 | log::trace!("exit: start"); 143 | Ok(()) 144 | } 145 | 146 | /// Wrapper around incrementing the overall scan's progress bar 147 | fn increment_bar(&self) { 148 | let msg = format!( 149 | "{}:{:<7} {}:{:<7}", 150 | style("found").green(), 151 | self.stats.resources_discovered(), 152 | style("errors").red(), 153 | self.stats.errors(), 154 | ); 155 | 156 | self.bar.set_message(msg); 157 | 158 | if self.bar.position() < self.stats.total_expected() as u64 { 159 | // don't run off the end when we're a few requests over the expected total 160 | // due to the heuristics tests 161 | self.bar.inc(1); 162 | } 163 | } 164 | 165 | /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for 166 | /// updates to the aforementioned object. 167 | pub fn initialize(config: Arc) -> (Joiner, StatsHandle) { 168 | log::trace!("enter: initialize"); 169 | 170 | let data = Arc::new(Stats::new(config.json)); 171 | let (tx, rx): FeroxChannel = mpsc::unbounded_channel(); 172 | 173 | let mut handler = StatsHandler::new(data.clone(), rx); 174 | 175 | let task = tokio::spawn(async move { handler.start(&config.output).await }); 176 | 177 | let event_handle = StatsHandle::new(data, tx); 178 | 179 | log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle); 180 | 181 | (task, event_handle) 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/extractor/builder.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::event_handlers::Handles; 3 | use anyhow::{bail, Result}; 4 | 5 | /// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder) 6 | /// 7 | /// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files) 8 | pub(super) const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#; 9 | 10 | /// Regular expression to pull url paths from robots.txt 11 | /// 12 | /// ref: https://developers.google.com/search/reference/robots_txt 13 | pub(super) const ROBOTS_TXT_REGEX: &str = 14 | r#"(?m)^ *(Allow|Disallow): *(?P[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m) 15 | 16 | /// Regular expression to filter bad characters from extracted url paths 17 | /// 18 | /// ref: https://www.rfc-editor.org/rfc/rfc3986#section-2 19 | pub(super) const URL_CHARS_REGEX: &str = r#"["<>\\^`{|} ]"#; 20 | 21 | /// Which type of extraction should be performed 22 | #[derive(Debug, Copy, Clone)] 23 | pub enum ExtractionTarget { 24 | /// Examine a response body and extract javascript and html links (multiple tags) 25 | ResponseBody, 26 | 27 | /// Examine robots.txt (specifically) and extract links 28 | RobotsTxt, 29 | 30 | /// Extract all tags from a page 31 | DirectoryListing, 32 | } 33 | 34 | /// responsible for building an `Extractor` 35 | pub struct ExtractorBuilder<'a> { 36 | /// Response from which to extract links 37 | response: Option<&'a FeroxResponse>, 38 | 39 | /// URL of where to extract links 40 | url: String, 41 | 42 | /// Handles object to house the underlying mpsc transmitters 43 | handles: Option>, 44 | 45 | /// type of extraction to be performed 46 | target: ExtractionTarget, 47 | } 48 | 49 | /// ExtractorBuilder implementation 50 | impl<'a> Default for ExtractorBuilder<'a> { 51 | fn default() -> Self { 52 | Self { 53 | response: None, 54 | url: "".to_string(), 55 | handles: None, 56 | target: ExtractionTarget::ResponseBody, 57 | } 58 | } 59 | } 60 | 61 | /// ExtractorBuilder implementation 62 | impl<'a> ExtractorBuilder<'a> { 63 | /// builder call to set `handles` 64 | pub fn handles(&mut self, handles: Arc) -> &mut Self { 65 | self.handles = Some(handles); 66 | self 67 | } 68 | 69 | /// builder call to set `url` 70 | pub fn url(&mut self, url: &str) -> &mut Self { 71 | self.url = url.to_string(); 72 | self 73 | } 74 | 75 | /// builder call to set `target` 76 | pub fn target(&mut self, target: ExtractionTarget) -> &mut Self { 77 | self.target = target; 78 | self 79 | } 80 | 81 | /// builder call to set `response` 82 | pub fn response(&mut self, response: &'a FeroxResponse) -> &mut Self { 83 | self.response = Some(response); 84 | self 85 | } 86 | 87 | /// finalize configuration of `ExtractorBuilder` and return an `Extractor` 88 | /// 89 | /// requires either `with_url` or `with_response` to have been used in the build process 90 | pub fn build(&self) -> Result> { 91 | if (self.url.is_empty() && self.response.is_none()) || self.handles.is_none() { 92 | bail!("Extractor requires a URL or a FeroxResponse be specified as well as a Handles object") 93 | } 94 | 95 | Ok(Extractor { 96 | links_regex: Regex::new(LINKFINDER_REGEX).unwrap(), 97 | robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(), 98 | url_regex: Regex::new(URL_CHARS_REGEX).unwrap(), 99 | response: if self.response.is_some() { 100 | Some(self.response.unwrap()) 101 | } else { 102 | None 103 | }, 104 | url: self.url.to_owned(), 105 | handles: self.handles.as_ref().unwrap().clone(), 106 | target: self.target, 107 | }) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/extractor/mod.rs: -------------------------------------------------------------------------------- 1 | //! extract links from html source and robots.txt 2 | mod builder; 3 | mod container; 4 | #[cfg(test)] 5 | mod tests; 6 | 7 | pub use self::builder::ExtractionTarget; 8 | pub use self::builder::ExtractorBuilder; 9 | pub use self::container::Extractor; 10 | 11 | use crate::response::FeroxResponse; 12 | use regex::Regex; 13 | use std::sync::Arc; 14 | -------------------------------------------------------------------------------- /src/filters/container.rs: -------------------------------------------------------------------------------- 1 | use std::sync::RwLock; 2 | 3 | use anyhow::Result; 4 | use serde::{ser::SerializeSeq, Serialize, Serializer}; 5 | 6 | use crate::response::FeroxResponse; 7 | 8 | use super::{ 9 | FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, 10 | WildcardFilter, WordsFilter, 11 | }; 12 | use crate::{ 13 | event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered, 14 | CommandSender, 15 | }; 16 | /// Container around a collection of `FeroxFilters`s 17 | #[derive(Debug, Default)] 18 | pub struct FeroxFilters { 19 | /// collection of `FeroxFilters` 20 | pub filters: RwLock>>, 21 | } 22 | 23 | /// implementation of FeroxFilter collection 24 | impl FeroxFilters { 25 | /// add a single FeroxFilter to the collection 26 | pub fn push(&self, filter: Box) -> Result<()> { 27 | if let Ok(mut guard) = self.filters.write() { 28 | if guard.contains(&filter) { 29 | return Ok(()); 30 | } 31 | 32 | guard.push(filter) 33 | } 34 | Ok(()) 35 | } 36 | 37 | /// remove items from the underlying collection by their index 38 | /// 39 | /// note: indexes passed in should be index-to-remove+1. This is built for the scan mgt menu 40 | /// so indexes aren't 0-based whehn the user enters them. 41 | /// 42 | pub fn remove(&self, indices: &mut [usize]) { 43 | // since we're removing by index, indices must be sorted and then reversed. 44 | // this allows us to iterate over the collection from the rear, allowing any shifting 45 | // of the vector to happen on sections that we no longer care about, as we're moving 46 | // in the opposite direction 47 | indices.sort_unstable(); 48 | indices.reverse(); 49 | 50 | if let Ok(mut guard) = self.filters.write() { 51 | for index in indices { 52 | // numbering of the menu starts at 1, so we'll need to reduce the index by 1 53 | // to account for that. if they've provided 0 as an offset, we'll set the 54 | // result to a gigantic number and skip it in the loop with a bounds check 55 | let reduced_idx = index.checked_sub(1).unwrap_or(usize::MAX); 56 | 57 | // check if number provided is out of range 58 | if reduced_idx >= guard.len() { 59 | // usize can't be negative, just need to handle exceeding bounds 60 | continue; 61 | } 62 | 63 | guard.remove(reduced_idx); 64 | } 65 | } 66 | } 67 | 68 | /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported 69 | /// to the user or not. 70 | pub fn should_filter_response( 71 | &self, 72 | response: &FeroxResponse, 73 | tx_stats: CommandSender, 74 | ) -> bool { 75 | if let Ok(filters) = self.filters.read() { 76 | for filter in filters.iter() { 77 | // wildcard.should_filter goes here 78 | if filter.should_filter_response(response) { 79 | log::debug!("filtering response due to: {:?}", filter); 80 | if filter.as_any().downcast_ref::().is_some() { 81 | tx_stats 82 | .send(AddToUsizeField(WildcardsFiltered, 1)) 83 | .unwrap_or_default(); 84 | } 85 | return true; 86 | } 87 | } 88 | } 89 | false 90 | } 91 | } 92 | 93 | impl Serialize for FeroxFilters { 94 | fn serialize(&self, serializer: S) -> Result 95 | where 96 | S: Serializer, 97 | { 98 | if let Ok(guard) = self.filters.read() { 99 | let mut seq = serializer.serialize_seq(Some(guard.len()))?; 100 | 101 | for filter in &*guard { 102 | if let Some(line_filter) = filter.as_any().downcast_ref::() { 103 | seq.serialize_element(line_filter).unwrap_or_default(); 104 | } else if let Some(word_filter) = filter.as_any().downcast_ref::() { 105 | seq.serialize_element(word_filter).unwrap_or_default(); 106 | } else if let Some(size_filter) = filter.as_any().downcast_ref::() { 107 | seq.serialize_element(size_filter).unwrap_or_default(); 108 | } else if let Some(wildcard_filter) = 109 | filter.as_any().downcast_ref::() 110 | { 111 | seq.serialize_element(wildcard_filter).unwrap_or_default(); 112 | } else if let Some(status_filter) = 113 | filter.as_any().downcast_ref::() 114 | { 115 | seq.serialize_element(status_filter).unwrap_or_default(); 116 | } else if let Some(regex_filter) = filter.as_any().downcast_ref::() { 117 | seq.serialize_element(regex_filter).unwrap_or_default(); 118 | } else if let Some(similarity_filter) = 119 | filter.as_any().downcast_ref::() 120 | { 121 | seq.serialize_element(similarity_filter).unwrap_or_default(); 122 | } 123 | } 124 | seq.end() 125 | } else { 126 | // if for some reason we can't unlock the mutex, just write an empty list 127 | let seq = serializer.serialize_seq(Some(0))?; 128 | seq.end() 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/filters/empty.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | /// Dummy filter for internal shenanigans 4 | #[derive(Default, Debug, PartialEq, Eq)] 5 | pub struct EmptyFilter {} 6 | 7 | impl FeroxFilter for EmptyFilter { 8 | /// `EmptyFilter` always returns false 9 | fn should_filter_response(&self, _response: &FeroxResponse) -> bool { 10 | false 11 | } 12 | 13 | /// Compare one EmptyFilter to another 14 | fn box_eq(&self, other: &dyn Any) -> bool { 15 | other.downcast_ref::().map_or(false, |a| self == a) 16 | } 17 | 18 | /// Return self as Any for dynamic dispatch purposes 19 | fn as_any(&self) -> &dyn Any { 20 | self 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/filters/init.rs: -------------------------------------------------------------------------------- 1 | use super::{ 2 | utils::create_similarity_filter, LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter, 3 | WordsFilter, 4 | }; 5 | use crate::{event_handlers::Handles, skip_fail, utils::fmt_err, Command::AddFilter}; 6 | use anyhow::Result; 7 | use regex::Regex; 8 | use std::sync::Arc; 9 | 10 | /// add all user-supplied filters to the (already started) filters handler 11 | pub async fn initialize(handles: Arc) -> Result<()> { 12 | // add any status code filters to filters handler's FeroxFilters (-C|--filter-status) 13 | for code_filter in &handles.config.filter_status { 14 | let filter = StatusCodeFilter { 15 | filter_code: *code_filter, 16 | }; 17 | let boxed_filter = Box::new(filter); 18 | skip_fail!(handles.filters.send(AddFilter(boxed_filter))); 19 | } 20 | 21 | // add any line count filters to filters handler's FeroxFilters (-N|--filter-lines) 22 | for lines_filter in &handles.config.filter_line_count { 23 | let filter = LinesFilter { 24 | line_count: *lines_filter, 25 | }; 26 | let boxed_filter = Box::new(filter); 27 | skip_fail!(handles.filters.send(AddFilter(boxed_filter))); 28 | } 29 | 30 | // add any line count filters to filters handler's FeroxFilters (-W|--filter-words) 31 | for words_filter in &handles.config.filter_word_count { 32 | let filter = WordsFilter { 33 | word_count: *words_filter, 34 | }; 35 | let boxed_filter = Box::new(filter); 36 | skip_fail!(handles.filters.send(AddFilter(boxed_filter))); 37 | } 38 | 39 | // add any line count filters to filters handler's FeroxFilters (-S|--filter-size) 40 | for size_filter in &handles.config.filter_size { 41 | let filter = SizeFilter { 42 | content_length: *size_filter, 43 | }; 44 | let boxed_filter = Box::new(filter); 45 | skip_fail!(handles.filters.send(AddFilter(boxed_filter))); 46 | } 47 | 48 | // add any regex filters to filters handler's FeroxFilters (-X|--filter-regex) 49 | for regex_filter in &handles.config.filter_regex { 50 | let raw = regex_filter; 51 | let compiled = skip_fail!(Regex::new(raw)); 52 | 53 | let filter = RegexFilter { 54 | raw_string: raw.to_owned(), 55 | compiled, 56 | }; 57 | let boxed_filter = Box::new(filter); 58 | skip_fail!(handles.filters.send(AddFilter(boxed_filter))); 59 | } 60 | 61 | // add any similarity filters to filters handler's FeroxFilters (--filter-similar-to) 62 | for similarity_filter in &handles.config.filter_similar { 63 | let filter = skip_fail!(create_similarity_filter(similarity_filter, handles.clone()).await); 64 | 65 | let boxed_filter = Box::new(filter); 66 | skip_fail!(handles.filters.send(AddFilter(boxed_filter))); 67 | } 68 | 69 | handles.filters.sync().await?; 70 | Ok(()) 71 | } 72 | -------------------------------------------------------------------------------- /src/filters/lines.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | /// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines 4 | /// in a Response body; specified using -N|--filter-lines 5 | #[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] 6 | pub struct LinesFilter { 7 | /// Number of lines in a Response's body that should be filtered 8 | pub line_count: usize, 9 | } 10 | 11 | /// implementation of FeroxFilter for LinesFilter 12 | impl FeroxFilter for LinesFilter { 13 | /// Check `line_count` against what was passed in via -N|--filter-lines 14 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 15 | log::trace!("enter: should_filter_response({:?} {})", self, response); 16 | 17 | let result = response.line_count() == self.line_count; 18 | 19 | log::trace!("exit: should_filter_response -> {}", result); 20 | 21 | result 22 | } 23 | 24 | /// Compare one LinesFilter to another 25 | fn box_eq(&self, other: &dyn Any) -> bool { 26 | other.downcast_ref::().map_or(false, |a| self == a) 27 | } 28 | 29 | /// Return self as Any for dynamic dispatch purposes 30 | fn as_any(&self) -> &dyn Any { 31 | self 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/filters/mod.rs: -------------------------------------------------------------------------------- 1 | //! contains all of feroxbuster's filters 2 | use serde::{Deserialize, Serialize}; 3 | use std::any::Any; 4 | use std::fmt::Debug; 5 | 6 | use crate::response::FeroxResponse; 7 | use crate::traits::FeroxFilter; 8 | 9 | pub use self::container::FeroxFilters; 10 | pub(crate) use self::empty::EmptyFilter; 11 | pub use self::init::initialize; 12 | pub use self::lines::LinesFilter; 13 | pub use self::regex::RegexFilter; 14 | pub use self::similarity::{SimilarityFilter, SIM_HASHER}; 15 | pub use self::size::SizeFilter; 16 | pub use self::status_code::StatusCodeFilter; 17 | pub(crate) use self::utils::{create_similarity_filter, filter_lookup}; 18 | pub use self::wildcard::WildcardFilter; 19 | pub use self::words::WordsFilter; 20 | 21 | mod status_code; 22 | mod words; 23 | mod lines; 24 | mod size; 25 | mod regex; 26 | mod similarity; 27 | mod container; 28 | #[cfg(test)] 29 | mod tests; 30 | mod init; 31 | mod utils; 32 | mod wildcard; 33 | mod empty; 34 | -------------------------------------------------------------------------------- /src/filters/regex.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use ::regex::Regex; 3 | 4 | /// Simple implementor of FeroxFilter; used to filter out responses based on a given regular 5 | /// expression; specified using -X|--filter-regex 6 | #[derive(Debug, Serialize, Deserialize)] 7 | pub struct RegexFilter { 8 | /// Regular expression to be applied to the response body for filtering, compiled 9 | #[serde(with = "serde_regex")] 10 | pub compiled: Regex, 11 | 12 | /// Regular expression as passed in on the command line, not compiled 13 | pub raw_string: String, 14 | } 15 | 16 | impl Default for RegexFilter { 17 | fn default() -> Self { 18 | Self { 19 | compiled: Regex::new("").unwrap(), 20 | raw_string: String::new(), 21 | } 22 | } 23 | } 24 | 25 | /// implementation of FeroxFilter for RegexFilter 26 | impl FeroxFilter for RegexFilter { 27 | /// Check `expression` against the response body, if the expression matches, the response 28 | /// should be filtered out 29 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 30 | log::trace!("enter: should_filter_response({:?} {})", self, response); 31 | 32 | let result = self.compiled.is_match(response.text()); 33 | let other = response.headers().iter().any(|(k, v)| { 34 | self.compiled.is_match(k.as_str()) || self.compiled.is_match(v.to_str().unwrap_or("")) 35 | }); 36 | 37 | log::trace!("exit: should_filter_response -> {}", result || other); 38 | 39 | result || other 40 | } 41 | 42 | /// Compare one SizeFilter to another 43 | fn box_eq(&self, other: &dyn Any) -> bool { 44 | other.downcast_ref::().map_or(false, |a| self == a) 45 | } 46 | 47 | /// Return self as Any for dynamic dispatch purposes 48 | fn as_any(&self) -> &dyn Any { 49 | self 50 | } 51 | } 52 | 53 | /// PartialEq implementation for RegexFilter 54 | impl PartialEq for RegexFilter { 55 | /// Simple comparison of the raw string passed in via the command line 56 | fn eq(&self, other: &RegexFilter) -> bool { 57 | self.raw_string == other.raw_string 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/filters/similarity.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::nlp::preprocess; 3 | use gaoya::simhash::{SimHash, SimHashBits, SimSipHasher64}; 4 | use lazy_static::lazy_static; 5 | 6 | lazy_static! { 7 | /// single instance of the sip hasher used in similarity filtering 8 | pub static ref SIM_HASHER: SimHash = 9 | SimHash::::new(SimSipHasher64::new(1, 2)); 10 | } 11 | 12 | /// maximum hamming distance allowed between two signatures 13 | /// 14 | /// ref: https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/33026.pdf 15 | /// section: 4.1 Choice of Parameters 16 | const MAX_HAMMING_DISTANCE: usize = 3; 17 | 18 | /// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a 19 | /// Response body with a known response; specified using --filter-similar-to 20 | #[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] 21 | pub struct SimilarityFilter { 22 | /// Hash of Response's body to be used during similarity comparison 23 | pub hash: u64, 24 | 25 | /// Url originally requested for the similarity filter 26 | pub original_url: String, 27 | } 28 | 29 | /// implementation of FeroxFilter for SimilarityFilter 30 | impl FeroxFilter for SimilarityFilter { 31 | /// Check `FeroxResponse::text` against what was requested from the site passed in via 32 | /// --filter-similar-to 33 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 34 | let other = SIM_HASHER.create_signature(preprocess(response.text()).iter()); 35 | self.hash.hamming_distance(&other) <= MAX_HAMMING_DISTANCE 36 | } 37 | 38 | /// Compare one SimilarityFilter to another 39 | fn box_eq(&self, other: &dyn Any) -> bool { 40 | other 41 | .downcast_ref::() 42 | .map_or(false, |a| self.hash == a.hash) 43 | } 44 | 45 | /// Return self as Any for dynamic dispatch purposes 46 | fn as_any(&self) -> &dyn Any { 47 | self 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/filters/size.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | /// Simple implementor of FeroxFilter; used to filter out responses based on the length of a 4 | /// Response body; specified using -S|--filter-size 5 | #[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] 6 | pub struct SizeFilter { 7 | /// Overall length of a Response's body that should be filtered 8 | pub content_length: u64, 9 | } 10 | 11 | /// implementation of FeroxFilter for SizeFilter 12 | impl FeroxFilter for SizeFilter { 13 | /// Check `content_length` against what was passed in via -S|--filter-size 14 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 15 | log::trace!("enter: should_filter_response({:?} {})", self, response); 16 | 17 | let result = response.content_length() == self.content_length; 18 | 19 | log::trace!("exit: should_filter_response -> {}", result); 20 | 21 | result 22 | } 23 | 24 | /// Compare one SizeFilter to another 25 | fn box_eq(&self, other: &dyn Any) -> bool { 26 | other.downcast_ref::().map_or(false, |a| self == a) 27 | } 28 | 29 | /// Return self as Any for dynamic dispatch purposes 30 | fn as_any(&self) -> &dyn Any { 31 | self 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/filters/status_code.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | /// Simple implementor of FeroxFilter; used to filter out status codes specified using 4 | /// -C|--filter-status 5 | #[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] 6 | pub struct StatusCodeFilter { 7 | /// Status code that should not be displayed to the user 8 | pub filter_code: u16, 9 | } 10 | 11 | /// implementation of FeroxFilter for StatusCodeFilter 12 | impl FeroxFilter for StatusCodeFilter { 13 | /// Check `filter_code` against what was passed in via -C|--filter-status 14 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 15 | log::trace!("enter: should_filter_response({:?} {})", self, response); 16 | 17 | if response.status().as_u16() == self.filter_code { 18 | log::debug!( 19 | "filtered out {} based on --filter-status of {}", 20 | response.url(), 21 | self.filter_code 22 | ); 23 | log::trace!("exit: should_filter_response -> true"); 24 | return true; 25 | } 26 | 27 | log::trace!("exit: should_filter_response -> false"); 28 | false 29 | } 30 | 31 | /// Compare one StatusCodeFilter to another 32 | fn box_eq(&self, other: &dyn Any) -> bool { 33 | other.downcast_ref::().map_or(false, |a| self == a) 34 | } 35 | 36 | /// Return self as Any for dynamic dispatch purposes 37 | fn as_any(&self) -> &dyn Any { 38 | self 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/filters/utils.rs: -------------------------------------------------------------------------------- 1 | use super::FeroxFilter; 2 | use super::SimilarityFilter; 3 | use crate::event_handlers::Handles; 4 | use crate::filters::similarity::SIM_HASHER; 5 | use crate::nlp::preprocess; 6 | use crate::response::FeroxResponse; 7 | use crate::utils::{logged_request, parse_url_with_raw_path}; 8 | use crate::DEFAULT_METHOD; 9 | use anyhow::Result; 10 | use regex::Regex; 11 | use std::sync::Arc; 12 | 13 | /// wrapper around logic necessary to create a SimilarityFilter 14 | /// 15 | /// - parses given url 16 | /// - makes request to the parsed url 17 | /// - gathers extensions from the url, if configured to do so 18 | /// - computes hash of response body 19 | /// - creates filter with hash 20 | pub(crate) async fn create_similarity_filter( 21 | similarity_filter: &str, 22 | handles: Arc, 23 | ) -> Result { 24 | // url as-is based on input, ignores user-specified url manipulation options (add-slash etc) 25 | let url = parse_url_with_raw_path(similarity_filter)?; 26 | 27 | // attempt to request the given url 28 | let resp = logged_request(&url, DEFAULT_METHOD, None, handles.clone()).await?; 29 | 30 | // if successful, create a filter based on the response's body 31 | let mut fr = FeroxResponse::from( 32 | resp, 33 | similarity_filter, 34 | DEFAULT_METHOD, 35 | handles.config.output_level, 36 | ) 37 | .await; 38 | 39 | if handles.config.collect_extensions { 40 | fr.parse_extension(handles.clone())?; 41 | } 42 | 43 | let hash = SIM_HASHER.create_signature(preprocess(fr.text()).iter()); 44 | 45 | Ok(SimilarityFilter { 46 | hash, 47 | original_url: similarity_filter.to_string(), 48 | }) 49 | } 50 | 51 | /// used in conjunction with the Scan Management Menu 52 | /// 53 | /// when a user uses the n[ew-filter] command in the menu, the two params are passed here for 54 | /// processing. 55 | /// 56 | /// an example command may be `new-filter lines 40`. `lines` and `40` are passed here as &str's 57 | /// 58 | /// once here, the type and value are used to create an appropriate FeroxFilter. If anything 59 | /// goes wrong during creation, None is returned. 60 | pub(crate) fn filter_lookup(filter_type: &str, filter_value: &str) -> Option> { 61 | match filter_type { 62 | "status" => { 63 | if let Ok(parsed) = filter_value.parse() { 64 | return Some(Box::new(super::StatusCodeFilter { 65 | filter_code: parsed, 66 | })); 67 | } 68 | } 69 | "lines" => { 70 | if let Ok(parsed) = filter_value.parse() { 71 | return Some(Box::new(super::LinesFilter { line_count: parsed })); 72 | } 73 | } 74 | "size" => { 75 | if let Ok(parsed) = filter_value.parse() { 76 | return Some(Box::new(super::SizeFilter { 77 | content_length: parsed, 78 | })); 79 | } 80 | } 81 | "words" => { 82 | if let Ok(parsed) = filter_value.parse() { 83 | return Some(Box::new(super::WordsFilter { word_count: parsed })); 84 | } 85 | } 86 | "regex" => { 87 | if let Ok(parsed) = Regex::new(filter_value) { 88 | return Some(Box::new(super::RegexFilter { 89 | compiled: parsed, 90 | raw_string: filter_value.to_string(), 91 | })); 92 | } 93 | } 94 | "similarity" => { 95 | return Some(Box::new(SimilarityFilter { 96 | hash: 0, 97 | original_url: filter_value.to_string(), 98 | })); 99 | } 100 | _ => (), 101 | } 102 | 103 | None 104 | } 105 | 106 | #[cfg(test)] 107 | mod tests { 108 | use super::*; 109 | use crate::config::Configuration; 110 | use crate::filters::{LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter, WordsFilter}; 111 | use crate::scan_manager::FeroxScans; 112 | use httpmock::Method::GET; 113 | use httpmock::MockServer; 114 | 115 | #[test] 116 | /// filter_lookup returns correct filters 117 | fn filter_lookup_returns_correct_filters() { 118 | let filter = filter_lookup("status", "200").unwrap(); 119 | assert_eq!( 120 | filter.as_any().downcast_ref::().unwrap(), 121 | &StatusCodeFilter { filter_code: 200 } 122 | ); 123 | 124 | let filter = filter_lookup("lines", "10").unwrap(); 125 | assert_eq!( 126 | filter.as_any().downcast_ref::().unwrap(), 127 | &LinesFilter { line_count: 10 } 128 | ); 129 | 130 | let filter = filter_lookup("size", "20").unwrap(); 131 | assert_eq!( 132 | filter.as_any().downcast_ref::().unwrap(), 133 | &SizeFilter { content_length: 20 } 134 | ); 135 | 136 | let filter = filter_lookup("words", "30").unwrap(); 137 | assert_eq!( 138 | filter.as_any().downcast_ref::().unwrap(), 139 | &WordsFilter { word_count: 30 } 140 | ); 141 | 142 | let filter = filter_lookup("regex", "stuff.*").unwrap(); 143 | let compiled = Regex::new("stuff.*").unwrap(); 144 | let raw_string = String::from("stuff.*"); 145 | assert_eq!( 146 | filter.as_any().downcast_ref::().unwrap(), 147 | &RegexFilter { 148 | compiled, 149 | raw_string 150 | } 151 | ); 152 | 153 | let filter = filter_lookup("similarity", "http://localhost").unwrap(); 154 | assert_eq!( 155 | filter.as_any().downcast_ref::().unwrap(), 156 | &SimilarityFilter { 157 | hash: 0, 158 | original_url: "http://localhost".to_string() 159 | } 160 | ); 161 | 162 | assert!(filter_lookup("non-existent", "").is_none()); 163 | } 164 | 165 | #[tokio::test(flavor = "multi_thread", worker_threads = 1)] 166 | /// ensure create_similarity_filter correctness of return value and side-effects 167 | async fn create_similarity_filter_is_correct() { 168 | let srv = MockServer::start(); 169 | 170 | let mock = srv.mock(|when, then| { 171 | when.method(GET).path("/"); 172 | then.status(200).body("this is a test"); 173 | }); 174 | 175 | let scans = FeroxScans::default(); 176 | let config = Configuration { 177 | collect_extensions: true, 178 | ..Default::default() 179 | }; 180 | 181 | let (test_handles, _) = Handles::for_testing(Some(Arc::new(scans)), Some(Arc::new(config))); 182 | 183 | let handles = Arc::new(test_handles); 184 | 185 | let filter = create_similarity_filter(&srv.url("/"), handles.clone()) 186 | .await 187 | .unwrap(); 188 | 189 | assert_eq!(mock.hits(), 1); 190 | 191 | assert_eq!( 192 | filter, 193 | SimilarityFilter { 194 | hash: 14897447612059286329, 195 | original_url: srv.url("/") 196 | } 197 | ); 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/filters/wildcard.rs: -------------------------------------------------------------------------------- 1 | use console::style; 2 | 3 | use super::*; 4 | use crate::utils::create_report_string; 5 | use crate::{config::OutputLevel, DEFAULT_METHOD}; 6 | 7 | /// Data holder for all relevant data needed when auto-filtering out wildcard responses 8 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 9 | pub struct WildcardFilter { 10 | /// The content-length of this response, if known 11 | pub content_length: Option, 12 | 13 | /// The number of lines contained in the body of this response, if known 14 | pub line_count: Option, 15 | 16 | /// The number of words contained in the body of this response, if known 17 | pub word_count: Option, 18 | 19 | /// method used in request that should be included with filters passed via runtime configuration 20 | pub method: String, 21 | 22 | /// the status code returned in the response 23 | pub status_code: u16, 24 | 25 | /// whether or not the user passed -D on the command line 26 | pub dont_filter: bool, 27 | } 28 | 29 | /// implementation of WildcardFilter 30 | impl WildcardFilter { 31 | /// given a boolean representing whether -D was used or not, create a new WildcardFilter 32 | pub fn new(dont_filter: bool) -> Self { 33 | Self { 34 | dont_filter, 35 | ..Default::default() 36 | } 37 | } 38 | } 39 | 40 | /// implement default that populates `method` with its default value 41 | impl Default for WildcardFilter { 42 | fn default() -> Self { 43 | Self { 44 | content_length: None, 45 | line_count: None, 46 | word_count: None, 47 | method: DEFAULT_METHOD.to_string(), 48 | status_code: 0, 49 | dont_filter: false, 50 | } 51 | } 52 | } 53 | 54 | /// implementation of FeroxFilter for WildcardFilter 55 | impl FeroxFilter for WildcardFilter { 56 | /// Examine size/words/lines and method to determine whether or not the response received 57 | /// is a wildcard response and therefore should be filtered out 58 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 59 | log::trace!("enter: should_filter_response({:?} {})", self, response); 60 | 61 | // quick return if dont_filter is set 62 | if self.dont_filter { 63 | // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all 64 | // for not filtering anything. As such, it should live in the implementation of 65 | // a wildcard filter 66 | return false; 67 | } 68 | 69 | if self.method != response.method().as_str() { 70 | // method's don't match, so this response should not be filtered out 71 | log::trace!("exit: should_filter_response -> false"); 72 | return false; 73 | } 74 | 75 | if self.status_code != response.status().as_u16() { 76 | // status codes don't match, so this response should not be filtered out 77 | log::trace!("exit: should_filter_response -> false"); 78 | return false; 79 | } 80 | 81 | // methods and status codes match at this point, just need to check the other fields 82 | 83 | match (self.content_length, self.word_count, self.line_count) { 84 | (Some(cl), Some(wc), Some(lc)) => { 85 | if cl == response.content_length() 86 | && wc == response.word_count() 87 | && lc == response.line_count() 88 | { 89 | log::debug!("filtered out {}", response.url()); 90 | log::trace!("exit: should_filter_response -> true"); 91 | return true; 92 | } 93 | } 94 | (Some(cl), Some(wc), None) => { 95 | if cl == response.content_length() && wc == response.word_count() { 96 | log::debug!("filtered out {}", response.url()); 97 | log::trace!("exit: should_filter_response -> true"); 98 | return true; 99 | } 100 | } 101 | (Some(cl), None, Some(lc)) => { 102 | if cl == response.content_length() && lc == response.line_count() { 103 | log::debug!("filtered out {}", response.url()); 104 | log::trace!("exit: should_filter_response -> true"); 105 | return true; 106 | } 107 | } 108 | (None, Some(wc), Some(lc)) => { 109 | if wc == response.word_count() && lc == response.line_count() { 110 | log::debug!("filtered out {}", response.url()); 111 | log::trace!("exit: should_filter_response -> true"); 112 | return true; 113 | } 114 | } 115 | (Some(cl), None, None) => { 116 | if cl == response.content_length() { 117 | log::debug!("filtered out {}", response.url()); 118 | log::trace!("exit: should_filter_response -> true"); 119 | return true; 120 | } 121 | } 122 | (None, Some(wc), None) => { 123 | if wc == response.word_count() { 124 | log::debug!("filtered out {}", response.url()); 125 | log::trace!("exit: should_filter_response -> true"); 126 | return true; 127 | } 128 | } 129 | (None, None, Some(lc)) => { 130 | if lc == response.line_count() { 131 | log::debug!("filtered out {}", response.url()); 132 | log::trace!("exit: should_filter_response -> true"); 133 | return true; 134 | } 135 | } 136 | (None, None, None) => { 137 | unreachable!("wildcard filter without any filters set"); 138 | } 139 | } 140 | 141 | log::trace!("exit: should_filter_response -> false"); 142 | false 143 | } 144 | 145 | /// Compare one WildcardFilter to another 146 | fn box_eq(&self, other: &dyn Any) -> bool { 147 | other.downcast_ref::().map_or(false, |a| self == a) 148 | } 149 | 150 | /// Return self as Any for dynamic dispatch purposes 151 | fn as_any(&self) -> &dyn Any { 152 | self 153 | } 154 | } 155 | 156 | impl std::fmt::Display for WildcardFilter { 157 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 158 | let msg = create_report_string( 159 | self.status_code.to_string().as_str(), 160 | self.method.as_str(), 161 | &self 162 | .line_count 163 | .map_or_else(|| "-".to_string(), |x| x.to_string()), 164 | &self 165 | .word_count 166 | .map_or_else(|| "-".to_string(), |x| x.to_string()), 167 | &self 168 | .content_length 169 | .map_or_else(|| "-".to_string(), |x| x.to_string()), 170 | &format!( 171 | "{} found {}-like response and created new filter; toggle off with {}", 172 | style("Auto-filtering").bright().green(), 173 | style("404").red(), 174 | style("--dont-filter").yellow() 175 | ), 176 | OutputLevel::Default, 177 | ); 178 | write!(f, "{}", msg) 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /src/filters/words.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | /// Simple implementor of FeroxFilter; used to filter out responses based on the number of words 4 | /// in a Response body; specified using -W|--filter-words 5 | #[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)] 6 | pub struct WordsFilter { 7 | /// Number of words in a Response's body that should be filtered 8 | pub word_count: usize, 9 | } 10 | 11 | /// implementation of FeroxFilter for WordsFilter 12 | impl FeroxFilter for WordsFilter { 13 | /// Check `word_count` against what was passed in via -W|--filter-words 14 | fn should_filter_response(&self, response: &FeroxResponse) -> bool { 15 | log::trace!("enter: should_filter_response({:?} {})", self, response); 16 | 17 | let result = response.word_count() == self.word_count; 18 | 19 | log::trace!("exit: should_filter_response -> {}", result); 20 | 21 | result 22 | } 23 | 24 | /// Compare one WordsFilter to another 25 | fn box_eq(&self, other: &dyn Any) -> bool { 26 | other.downcast_ref::().map_or(false, |a| self == a) 27 | } 28 | 29 | /// Return self as Any for dynamic dispatch purposes 30 | fn as_any(&self) -> &dyn Any { 31 | self 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/logger.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::fs::OpenOptions; 3 | use std::io::BufWriter; 4 | use std::sync::{Arc, RwLock}; 5 | use std::time::Instant; 6 | 7 | use anyhow::{Context, Result}; 8 | use env_logger::Builder; 9 | 10 | use crate::{ 11 | config::Configuration, 12 | message::FeroxMessage, 13 | progress::PROGRESS_PRINTER, 14 | traits::FeroxSerialize, 15 | utils::{fmt_err, write_to}, 16 | }; 17 | 18 | /// Create a customized instance of 19 | /// [env_logger::Logger](https://docs.rs/env_logger/latest/env_logger/struct.Logger.html) 20 | /// with timer offset/color and set the log level based on `verbosity` 21 | pub fn initialize(config: Arc) -> Result<()> { 22 | // use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set 23 | // log level for the application; respects already specified RUST_LOG environment variable 24 | match env::var("RUST_LOG") { 25 | Ok(_) => {} // RUST_LOG found, don't override 26 | Err(_) => { 27 | // only set log level based on verbosity when RUST_LOG variable doesn't exist 28 | match config.verbosity { 29 | 0 => (), 30 | 1 => env::set_var("RUST_LOG", "warn"), 31 | 2 => env::set_var("RUST_LOG", "info"), 32 | 3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"), 33 | _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"), 34 | } 35 | } 36 | } 37 | 38 | let start = Instant::now(); 39 | let mut builder = Builder::from_default_env(); 40 | 41 | let file = if !config.debug_log.is_empty() { 42 | let f = OpenOptions::new() // std fs 43 | .create(true) 44 | .append(true) 45 | .open(&config.debug_log) 46 | .with_context(|| fmt_err(&format!("Could not open {}", &config.debug_log)))?; 47 | 48 | let mut writer = BufWriter::new(f); 49 | 50 | // write out the configuration to the debug file if it exists 51 | write_to(&*config, &mut writer, config.json)?; 52 | 53 | Some(Arc::new(RwLock::new(writer))) 54 | } else { 55 | None 56 | }; 57 | 58 | builder 59 | .format(move |_, record| { 60 | let log_entry = FeroxMessage { 61 | message: record.args().to_string(), 62 | level: record.level().to_string(), 63 | time_offset: start.elapsed().as_secs_f32(), 64 | module: record.target().to_string(), 65 | kind: "log".to_string(), 66 | }; 67 | 68 | PROGRESS_PRINTER.println(log_entry.as_str()); 69 | 70 | if let Some(buffered_file) = file.clone() { 71 | if let Ok(mut unlocked) = buffered_file.write() { 72 | let _ = write_to(&log_entry, &mut unlocked, config.json); 73 | } 74 | } 75 | 76 | Ok(()) 77 | }) 78 | .init(); 79 | 80 | Ok(()) 81 | } 82 | -------------------------------------------------------------------------------- /src/macros.rs: -------------------------------------------------------------------------------- 1 | #![macro_use] 2 | 3 | #[macro_export] 4 | /// wrapper to improve code readability 5 | macro_rules! send_command { 6 | ($tx:expr, $value:expr) => { 7 | $tx.send($value).unwrap_or_default(); 8 | }; 9 | } 10 | 11 | #[macro_export] 12 | /// while looping, check for a Result, if Ok return the value, if Err, continue 13 | macro_rules! skip_fail { 14 | ($res:expr) => { 15 | match $res { 16 | Ok(val) => val, 17 | Err(e) => { 18 | log::warn!("{}", fmt_err(&format!("{}; skipping...", e))); 19 | continue; 20 | } 21 | } 22 | }; 23 | } 24 | -------------------------------------------------------------------------------- /src/message.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | use console::{style, Color}; 3 | use serde::{Deserialize, Serialize}; 4 | 5 | use crate::traits::FeroxSerialize; 6 | use crate::utils::fmt_err; 7 | 8 | #[derive(Serialize, Deserialize, Default, Debug)] 9 | /// Representation of a log entry, can be represented as a human readable string or JSON 10 | pub struct FeroxMessage { 11 | #[serde(rename = "type")] 12 | /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}` 13 | pub(crate) kind: String, 14 | 15 | /// The log message 16 | pub(crate) message: String, 17 | 18 | /// The log level 19 | pub(crate) level: String, 20 | 21 | /// The number of seconds elapsed since the scan started 22 | pub(crate) time_offset: f32, 23 | 24 | /// The module from which log::* was called 25 | pub(crate) module: String, 26 | } 27 | 28 | /// Implementation of FeroxMessage 29 | impl FeroxSerialize for FeroxMessage { 30 | /// Create a string representation of the log message 31 | /// 32 | /// ex: 301 10l 16w 173c https://localhost/api 33 | fn as_str(&self) -> String { 34 | let (level_name, level_color) = match self.level.as_str() { 35 | "ERROR" => ("ERR", Color::Red), 36 | "WARN" => ("WRN", Color::Red), 37 | "INFO" => ("INF", Color::Cyan), 38 | "DEBUG" => ("DBG", Color::Yellow), 39 | "TRACE" => ("TRC", Color::Magenta), 40 | "WILDCARD" => ("WLD", Color::Cyan), 41 | _ => ("MSG", Color::White), 42 | }; 43 | 44 | format!( 45 | "{} {:10.03} {} {}\n", 46 | style(level_name).bg(level_color).black(), 47 | style(self.time_offset).dim(), 48 | self.module, 49 | style(&self.message).dim(), 50 | ) 51 | } 52 | 53 | /// Create an NDJSON representation of the log message 54 | /// 55 | /// (expanded for clarity) 56 | /// ex: 57 | /// { 58 | /// "type": "log", 59 | /// "message": "Sent https://localhost/api to file handler", 60 | /// "level": "DEBUG", 61 | /// "time_offset": 0.86333454, 62 | /// "module": "feroxbuster::reporter" 63 | /// }\n 64 | fn as_json(&self) -> anyhow::Result { 65 | let mut json = serde_json::to_string(&self).with_context(|| { 66 | fmt_err(&format!( 67 | "Could not convert {}:{} to JSON", 68 | self.level, self.message 69 | )) 70 | })?; 71 | json.push('\n'); 72 | Ok(json) 73 | } 74 | } 75 | 76 | #[cfg(test)] 77 | mod tests { 78 | use super::*; 79 | 80 | #[test] 81 | /// test as_str method of FeroxMessage 82 | fn ferox_message_as_str_returns_string_with_newline() { 83 | let message = FeroxMessage { 84 | message: "message".to_string(), 85 | module: "utils".to_string(), 86 | time_offset: 1.0, 87 | level: "INFO".to_string(), 88 | kind: "log".to_string(), 89 | }; 90 | let message_str = message.as_str(); 91 | 92 | assert!(message_str.contains("INF")); 93 | assert!(message_str.contains("1.000")); 94 | assert!(message_str.contains("utils")); 95 | assert!(message_str.contains("message")); 96 | assert!(message_str.ends_with('\n')); 97 | } 98 | 99 | #[test] 100 | /// test as_json method of FeroxMessage 101 | fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() { 102 | let message = FeroxMessage { 103 | message: "message".to_string(), 104 | module: "utils".to_string(), 105 | time_offset: 1.0, 106 | level: "INFO".to_string(), 107 | kind: "log".to_string(), 108 | }; 109 | 110 | let message_str = message.as_json().unwrap(); 111 | 112 | let error_margin = f32::EPSILON; 113 | 114 | let json: FeroxMessage = serde_json::from_str(&message_str).unwrap(); 115 | assert_eq!(json.module, message.module); 116 | assert_eq!(json.message, message.message); 117 | assert!((json.time_offset - message.time_offset).abs() < error_margin); 118 | assert_eq!(json.level, message.level); 119 | assert_eq!(json.kind, message.kind); 120 | } 121 | 122 | #[test] 123 | /// test defaults for coverage 124 | fn message_defaults() { 125 | let msg = FeroxMessage::default(); 126 | assert_eq!(msg.level, String::new()); 127 | assert_eq!(msg.kind, String::new()); 128 | assert_eq!(msg.message, String::new()); 129 | assert_eq!(msg.module, String::new()); 130 | assert!(msg.time_offset < 0.1); 131 | } 132 | 133 | #[test] 134 | /// ensure WILDCARD messages serialize to WLD and anything not known to UNK 135 | fn message_as_str_edges() { 136 | let mut msg = FeroxMessage { 137 | message: "message".to_string(), 138 | module: "utils".to_string(), 139 | time_offset: 1.0, 140 | level: "WILDCARD".to_string(), 141 | kind: "log".to_string(), 142 | }; 143 | assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD")); 144 | 145 | msg.level = "UNKNOWN".to_string(); 146 | assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("MSG")); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/nlp/constants.rs: -------------------------------------------------------------------------------- 1 | use lazy_static::lazy_static; 2 | use regex::Regex; 3 | 4 | lazy_static! { 5 | /// regular expression to match on words with numbers, underscores, and hyphens 6 | pub(super) static ref BOUNDED_WORD_REGEX: Regex = Regex::new(r"\b[a-zA-Z0-9_-]+\b").unwrap(); 7 | } 8 | 9 | /// collection of stop words from spaCy with small modifications 10 | pub(super) static STOP_WORDS: [&str; 323] = [ 11 | "'d", 12 | "'ll", 13 | "'m", 14 | "'re", 15 | "'s", 16 | "'ve", 17 | "a", 18 | "about", 19 | "above", 20 | "across", 21 | "after", 22 | "afterwards", 23 | "again", 24 | "against", 25 | "almost", 26 | "alone", 27 | "along", 28 | "already", 29 | "also", 30 | "although", 31 | "always", 32 | "am", 33 | "among", 34 | "amongst", 35 | "amount", 36 | "an", 37 | "and", 38 | "another", 39 | "any", 40 | "anyhow", 41 | "anyone", 42 | "anything", 43 | "anyway", 44 | "anywhere", 45 | "are", 46 | "around", 47 | "as", 48 | "at", 49 | "back", 50 | "be", 51 | "became", 52 | "because", 53 | "become", 54 | "becomes", 55 | "becoming", 56 | "been", 57 | "before", 58 | "beforehand", 59 | "behind", 60 | "being", 61 | "below", 62 | "beside", 63 | "besides", 64 | "between", 65 | "beyond", 66 | "both", 67 | "bottom", 68 | "but", 69 | "by", 70 | "ca", 71 | "call", 72 | "can", 73 | "cannot", 74 | "could", 75 | "did", 76 | "do", 77 | "does", 78 | "doing", 79 | "done", 80 | "down", 81 | "due", 82 | "during", 83 | "each", 84 | "eight", 85 | "either", 86 | "eleven", 87 | "else", 88 | "elsewhere", 89 | "empty", 90 | "enough", 91 | "even", 92 | "ever", 93 | "every", 94 | "everyone", 95 | "everything", 96 | "everywhere", 97 | "except", 98 | "few", 99 | "fifteen", 100 | "fifty", 101 | "first", 102 | "five", 103 | "for", 104 | "former", 105 | "formerly", 106 | "forty", 107 | "four", 108 | "from", 109 | "front", 110 | "full", 111 | "further", 112 | "get", 113 | "got", 114 | "give", 115 | "go", 116 | "had", 117 | "has", 118 | "have", 119 | "he", 120 | "hence", 121 | "her", 122 | "here", 123 | "hereafter", 124 | "hereby", 125 | "herein", 126 | "hereupon", 127 | "hers", 128 | "herself", 129 | "him", 130 | "himself", 131 | "his", 132 | "how", 133 | "however", 134 | "hundred", 135 | "i", 136 | "if", 137 | "in", 138 | "indeed", 139 | "into", 140 | "is", 141 | "it", 142 | "its", 143 | "itself", 144 | "just", 145 | "keep", 146 | "last", 147 | "latter", 148 | "latterly", 149 | "least", 150 | "less", 151 | "made", 152 | "make", 153 | "many", 154 | "may", 155 | "me", 156 | "meanwhile", 157 | "might", 158 | "mine", 159 | "more", 160 | "moreover", 161 | "most", 162 | "mostly", 163 | "move", 164 | "much", 165 | "must", 166 | "my", 167 | "myself", 168 | "n't", 169 | "name", 170 | "namely", 171 | "neither", 172 | "never", 173 | "nevertheless", 174 | "next", 175 | "nine", 176 | "no", 177 | "nobody", 178 | "none", 179 | "noone", 180 | "nor", 181 | "not", 182 | "nothing", 183 | "now", 184 | "nowhere", 185 | "n\u{2018}t", 186 | "n\u{2019}t", 187 | "of", 188 | "off", 189 | "often", 190 | "on", 191 | "once", 192 | "one", 193 | "only", 194 | "onto", 195 | "or", 196 | "other", 197 | "others", 198 | "otherwise", 199 | "our", 200 | "ours", 201 | "ourselves", 202 | "out", 203 | "over", 204 | "own", 205 | "part", 206 | "per", 207 | "perhaps", 208 | "please", 209 | "put", 210 | "quite", 211 | "rather", 212 | "re", 213 | "really", 214 | "regarding", 215 | "same", 216 | "say", 217 | "see", 218 | "seem", 219 | "seemed", 220 | "seeming", 221 | "seems", 222 | "serious", 223 | "several", 224 | "she", 225 | "should", 226 | "side", 227 | "since", 228 | "six", 229 | "sixty", 230 | "so", 231 | "some", 232 | "somehow", 233 | "someone", 234 | "something", 235 | "sometime", 236 | "sometimes", 237 | "somewhere", 238 | "still", 239 | "such", 240 | "take", 241 | "ten", 242 | "than", 243 | "that", 244 | "the", 245 | "their", 246 | "them", 247 | "themselves", 248 | "then", 249 | "thence", 250 | "there", 251 | "thereafter", 252 | "thereby", 253 | "therefore", 254 | "therein", 255 | "thereupon", 256 | "these", 257 | "they", 258 | "third", 259 | "this", 260 | "those", 261 | "though", 262 | "three", 263 | "through", 264 | "throughout", 265 | "thru", 266 | "thus", 267 | "to", 268 | "together", 269 | "too", 270 | "toward", 271 | "towards", 272 | "twelve", 273 | "twenty", 274 | "two", 275 | "under", 276 | "unless", 277 | "until", 278 | "up", 279 | "upon", 280 | "used", 281 | "using", 282 | "various", 283 | "very", 284 | "via", 285 | "was", 286 | "we", 287 | "well", 288 | "were", 289 | "what", 290 | "whatever", 291 | "when", 292 | "whence", 293 | "whenever", 294 | "where", 295 | "whereafter", 296 | "whereas", 297 | "whereby", 298 | "wherein", 299 | "whereupon", 300 | "wherever", 301 | "whether", 302 | "which", 303 | "while", 304 | "whither", 305 | "who", 306 | "whoever", 307 | "whole", 308 | "whom", 309 | "whose", 310 | "why", 311 | "will", 312 | "with", 313 | "within", 314 | "without", 315 | "would", 316 | "yet", 317 | "you", 318 | "your", 319 | "yours", 320 | "yourself", 321 | "yourselves", 322 | "\u{2018}d", 323 | "\u{2018}ll", 324 | "\u{2018}m", 325 | "\u{2018}re", 326 | "\u{2018}s", 327 | "\u{2018}ve", 328 | "\u{2019}d", 329 | "\u{2019}ll", 330 | "\u{2019}m", 331 | "\u{2019}re", 332 | "\u{2019}s", 333 | "\u{2019}ve", 334 | ]; 335 | -------------------------------------------------------------------------------- /src/nlp/mod.rs: -------------------------------------------------------------------------------- 1 | //! small stand-alone tf-idf library, specifically designed for use in feroxbuster 2 | 3 | mod constants; 4 | mod document; 5 | mod model; 6 | mod term; 7 | mod utils; 8 | 9 | pub(crate) use self::document::Document; 10 | pub(crate) use self::model::TfIdf; 11 | pub(crate) use self::utils::preprocess; 12 | -------------------------------------------------------------------------------- /src/nlp/model.rs: -------------------------------------------------------------------------------- 1 | use super::document::Document; 2 | use super::term::{Term, TermMetaData}; 3 | use super::utils::{inverse_document_frequency, tf_idf_score}; 4 | use std::borrow::{Borrow, BorrowMut}; 5 | use std::collections::HashMap; 6 | 7 | /// data container for the TF-IDF model 8 | #[derive(Debug, Default)] 9 | pub(crate) struct TfIdf { 10 | /// collection of `Term`s and their associated metadata 11 | terms: HashMap, 12 | 13 | /// number of documents processed by the model 14 | num_documents: usize, 15 | } 16 | 17 | impl TfIdf { 18 | /// create an empty TF-IDF model; must be populated with `add_document` prior to use 19 | pub(crate) fn new() -> Self { 20 | Self::default() 21 | } 22 | 23 | /// accessor method for the collection of `Term`s and `TermMetaData` 24 | fn terms(&self) -> &HashMap { 25 | self.terms.borrow() 26 | } 27 | 28 | /// accessor method for the number of `Document`s the model has processed 29 | pub(crate) fn num_documents(&self) -> usize { 30 | self.num_documents 31 | } 32 | 33 | /// add a `Document` to the model 34 | pub(crate) fn add_document(&mut self, document: Document) { 35 | // increment number of docs seen, since we don't preserve the document itself; this needs 36 | // to happen before calls to `self.inverse_document_frequency`, as it relies on the count 37 | // being up to date 38 | self.num_documents += 1; 39 | 40 | for (term, doc_metadata) in document.terms().iter() { 41 | // an incoming `Term` from a `Document` only has a valid `count` for that particular 42 | // document; need to get the term frequency while both are known/valid 43 | let term_frequency = document.term_frequency(term); 44 | 45 | let metadata = self 46 | .terms 47 | .entry(term.clone()) 48 | .or_insert_with(|| doc_metadata.to_owned()); 49 | 50 | metadata.term_frequencies_mut().push(term_frequency); 51 | } 52 | } 53 | 54 | /// (re)-calculate tf-idf scores for all terms, given the current number of documents 55 | /// 56 | /// # Notes 57 | /// 58 | /// old tf-idf scores are removed during calculations to keep new `Term`s at the same relative 59 | /// level as new ones WRT corpus size 60 | pub(crate) fn calculate_tf_idf_scores(&mut self) { 61 | for metadata in self.terms.borrow_mut().values_mut() { 62 | let num_frequencies = metadata.term_frequencies().len(); 63 | 64 | let mut to_add = Vec::with_capacity(num_frequencies); 65 | 66 | for frequency in metadata.term_frequencies() { 67 | let idf = inverse_document_frequency( 68 | self.num_documents as f32, 69 | metadata.document_frequency() as f32, 70 | ); 71 | 72 | let score = tf_idf_score(*frequency, idf); 73 | to_add.push(score); 74 | } 75 | 76 | let average: f32 = to_add.iter().sum::() / to_add.len() as f32; 77 | 78 | *metadata.tf_idf_score_mut() = average; 79 | } 80 | } 81 | 82 | /// select all terms with a non-zero tf-idf score 83 | pub(crate) fn all_words(&self) -> Vec { 84 | self.terms() 85 | .iter() 86 | .filter(|(_, metadata)| metadata.tf_idf_score() > 0.0) 87 | .map(|(term, _)| term.raw().to_owned()) 88 | .collect() 89 | } 90 | } 91 | 92 | #[cfg(test)] 93 | mod tests { 94 | use super::*; 95 | 96 | /// helper for this test suite 97 | fn get_score(word: &str, model: &TfIdf) -> f32 { 98 | model.terms().get(&Term::new(word)).unwrap().tf_idf_score() 99 | } 100 | 101 | #[test] 102 | /// given the example data at https://remykarem.github.io/tfidf-demo/, ensure the model 103 | /// produces the same results 104 | fn model_generates_expected_tf_idf_scores() { 105 | let one = "Air quality in the sunny island improved gradually throughout Wednesday."; 106 | let two = 107 | "Air quality in Singapore on Wednesday continued to get worse as haze hit the island."; 108 | let three = "The air quality in Singapore is monitored through a network of air monitoring stations located in different parts of the island"; 109 | let four = "The air quality in Singapore got worse on Wednesday."; 110 | 111 | let docs = [one, two, three, four]; 112 | let mut model = TfIdf::new(); 113 | 114 | for doc in docs.iter() { 115 | let d = Document::new(doc); 116 | model.add_document(d); 117 | } 118 | 119 | assert_eq!(model.terms().len(), 19); 120 | 121 | model.calculate_tf_idf_scores(); 122 | 123 | assert_eq!(get_score("quality", &model), 0.0); 124 | assert_eq!(get_score("air", &model), 0.0); 125 | assert_eq!(get_score("wednesday", &model), 0.018906077); 126 | assert_eq!(get_score("island", &model), 0.014047348); 127 | assert_eq!(get_score("singapore", &model), 0.016427131); 128 | assert_eq!(get_score("sunny", &model), 0.08600858); 129 | assert_eq!(get_score("monitoring", &model), 0.05017167); 130 | assert_eq!(get_score("stations", &model), 0.05017167); 131 | assert_eq!(get_score("parts", &model), 0.05017167); 132 | assert_eq!(get_score("haze", &model), 0.06689556); 133 | assert_eq!(get_score("hit", &model), 0.06689556); 134 | assert_eq!(get_score("worse", &model), 0.04682689); 135 | } 136 | 137 | #[test] 138 | /// given the example data at https://remykarem.github.io/tfidf-demo/, ensure the model 139 | /// produces the same results 140 | fn select_n_words_grabs_correct_words() { 141 | let one = "Air quality in the sunny island improved gradually throughout Wednesday."; 142 | let two = 143 | "Air quality in Singapore on Wednesday continued to get worse as haze hit the island."; 144 | let three = "The air quality in Singapore is monitored through a network of air monitoring stations located in different parts of the island"; 145 | let four = "The air quality in Singapore got worse on Wednesday."; 146 | 147 | let docs = [one, two, three, four]; 148 | let mut model = TfIdf::new(); 149 | 150 | for doc in docs.iter() { 151 | let d = Document::new(doc); 152 | model.add_document(d); 153 | } 154 | 155 | assert_eq!(model.num_documents(), 4); 156 | 157 | model.calculate_tf_idf_scores(); 158 | 159 | let non_zero_words = model.all_words(); 160 | 161 | [ 162 | "gradually", 163 | "network", 164 | "hit", 165 | "located", 166 | "continued", 167 | "island", 168 | "worse", 169 | "monitored", 170 | "monitoring", 171 | "haze", 172 | "different", 173 | "stations", 174 | "sunny", 175 | "singapore", 176 | "improved", 177 | "parts", 178 | "wednesday", 179 | ] 180 | .iter() 181 | .for_each(|word| { 182 | assert!(non_zero_words.contains(&word.to_string())); 183 | }); 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/nlp/term.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::BorrowMut; 2 | 3 | /// single word term for text processing 4 | #[derive(Debug, Hash, Eq, PartialEq, Default, Clone)] 5 | pub(crate) struct Term { 6 | /// underlying string that the term represents 7 | raw: String, 8 | } 9 | 10 | impl Term { 11 | /// given a word, create a new `Term` 12 | pub(super) fn new(word: &str) -> Self { 13 | Self { 14 | raw: word.to_owned(), 15 | } 16 | } 17 | 18 | /// return a reference to the underlying string 19 | pub(super) fn raw(&self) -> &str { 20 | &self.raw 21 | } 22 | } 23 | 24 | /// metadata to be associated with a `Term` 25 | #[derive(Debug, Clone, Default)] 26 | pub(super) struct TermMetaData { 27 | /// number of times the associated `Term` was seen in a single document 28 | count: u32, 29 | 30 | /// collection of term frequencies for the associated `Term` 31 | term_frequencies: Vec, 32 | 33 | /// tf-idf score for the associated `Term` 34 | tf_idf_score: f32, 35 | } 36 | 37 | impl TermMetaData { 38 | /// number of times a `Term` has appeared in any `Document` within the corpus 39 | pub(super) fn document_frequency(&self) -> usize { 40 | self.term_frequencies().len() 41 | } 42 | 43 | /// mutable reference to the collection of term frequencies 44 | pub(super) fn term_frequencies_mut(&mut self) -> &mut Vec { 45 | self.term_frequencies.borrow_mut() 46 | } 47 | 48 | /// immutable reference to the collection of term frequencies 49 | pub(super) fn term_frequencies(&self) -> &[f32] { 50 | &self.term_frequencies 51 | } 52 | 53 | /// mutable reference to the number of times a `Term` was seen in a particular `Document` 54 | pub(super) fn count_mut(&mut self) -> &mut u32 { 55 | self.count.borrow_mut() 56 | } 57 | 58 | /// number of times a `Term` was seen in a particular `Document` 59 | pub(super) fn count(&self) -> u32 { 60 | self.count 61 | } 62 | 63 | /// mutable reference to the term's tf-idf score 64 | pub(super) fn tf_idf_score_mut(&mut self) -> &mut f32 { 65 | self.tf_idf_score.borrow_mut() 66 | } 67 | 68 | /// immutable reference to the term's tf-idf score 69 | pub(super) fn tf_idf_score(&self) -> f32 { 70 | self.tf_idf_score 71 | } 72 | } 73 | 74 | #[cfg(test)] 75 | mod tests { 76 | use super::*; 77 | 78 | #[test] 79 | /// test accessors for correctness 80 | fn nlp_term_accessor_test() { 81 | let term = Term::new("stuff"); 82 | assert_eq!(term.raw(), "stuff"); 83 | } 84 | 85 | #[test] 86 | /// test accessors for correctness 87 | fn nlp_term_metadata_accessor_test() { 88 | let mut metadata = TermMetaData::default(); 89 | 90 | *metadata.count_mut() += 1; 91 | assert_eq!(metadata.count(), 1); 92 | 93 | metadata.term_frequencies_mut().push(1.0); 94 | assert_eq!(metadata.document_frequency(), 1); 95 | assert_eq!(metadata.term_frequencies().first().unwrap(), &1.0); 96 | 97 | *metadata.tf_idf_score_mut() = 1.0_f32; 98 | assert_eq!(metadata.tf_idf_score(), 1.0); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/nlp/utils.rs: -------------------------------------------------------------------------------- 1 | use super::constants::{BOUNDED_WORD_REGEX, STOP_WORDS}; 2 | use regex::Captures; 3 | use std::borrow::Cow; 4 | 5 | /// pre-processing pipeline wrapper that removes punctuation, normalizes word case (utf-8 included) 6 | /// to lowercase, and remove stop words 7 | pub(crate) fn preprocess(text: &str) -> Vec { 8 | let text = remove_punctuation(text); 9 | let text = normalize_case(text); 10 | let text = remove_stop_words(&text); 11 | 12 | text.split_whitespace() 13 | .map(|word| word.to_string()) 14 | .collect::>() 15 | } 16 | 17 | /// optimized version of `str::to_lowercase` 18 | fn normalize_case<'a, S: Into>>(input: S) -> Cow<'a, str> { 19 | let input = input.into(); 20 | 21 | let first = input.find(char::is_uppercase); 22 | 23 | if let Some(first_idx) = first { 24 | let mut output = String::from(&input[..first_idx]); 25 | output.reserve(input.len() - first_idx); 26 | 27 | for c in input[first_idx..].chars() { 28 | if c.is_uppercase() { 29 | output.push(c.to_lowercase().next().unwrap()) 30 | } else { 31 | output.push(c) 32 | } 33 | } 34 | 35 | Cow::Owned(output) 36 | } else { 37 | input 38 | } 39 | } 40 | 41 | /// replace ascii and some utf-8 punctuation characters with ' ' (space) in the given string 42 | fn remove_punctuation(text: &str) -> String { 43 | text.replace( 44 | [ 45 | '!', '\\', '"', '#', '$', '%', '&', '(', ')', '*', '+', ':', ';', '<', '=', '>', '?', 46 | '@', '[', ']', '^', '{', '}', '|', '~', ',', '\'', '“', '”', '’', '‘', '’', '‘', '/', 47 | '–', '—', '.', 48 | ], 49 | " ", 50 | ) 51 | } 52 | 53 | /// remove stop words from the given string 54 | fn remove_stop_words(text: &str) -> String { 55 | BOUNDED_WORD_REGEX 56 | .replace_all(text, |caps: &Captures| { 57 | let word = &caps[0]; 58 | if !STOP_WORDS.contains(&word) { 59 | word.to_owned() 60 | } else { 61 | String::new() 62 | } 63 | }) 64 | .into() 65 | } 66 | 67 | /// calculate inverse document frequency 68 | pub(super) fn inverse_document_frequency(num_docs: f32, doc_frequency: f32) -> f32 { 69 | f32::log10(num_docs / doc_frequency) 70 | } 71 | 72 | /// calculate term frequency-inverse document frequency (tf-idf) 73 | pub(super) fn tf_idf_score(term_frequency: f32, idf: f32) -> f32 { 74 | term_frequency * idf 75 | } 76 | 77 | #[cfg(test)] 78 | mod tests { 79 | use super::*; 80 | 81 | #[test] 82 | /// ensure all expected punctuation characters are removed 83 | fn test_remove_punctuation() { 84 | let tester = "!\\\"#$%&()*+/:;<=>?@[]^{}|~,.'“”’‘–—\n‘’"; 85 | // the `" \n"` is because of the things like / getting replaced with a space 86 | assert_eq!( 87 | remove_punctuation(tester), 88 | " \n " 89 | ); 90 | } 91 | 92 | #[test] 93 | /// ensure uppercase characters are swapped to lowercase 94 | fn test_normalize_case() { 95 | let tester = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 96 | assert_eq!(normalize_case(tester), "abcdefghijklmnopqrstuvwxyz"); 97 | } 98 | 99 | #[test] 100 | /// ensure all stop words are removed from the list of stopwords ... intestuous 101 | fn test_remove_stopwords() { 102 | let all_words = STOP_WORDS 103 | .iter() 104 | .map(|&word| word.to_string()) 105 | .collect::>() 106 | .join(" "); 107 | 108 | let removed = remove_stop_words(&all_words).replace(' ', ""); 109 | 110 | // the remaining chars are from the contraction-based stop words 111 | assert_eq!(removed, "'d'll'm''s'ven'tn‘tn’t‘d‘ll‘m‘‘s‘ve’d’ll’m’’s’ve"); 112 | } 113 | 114 | #[test] 115 | /// ensure preprocess 116 | fn test_preprocess_results() { 117 | let tester = "WHY are Y'all YELLing?"; 118 | assert_eq!(&preprocess(tester), &["y", "all", "yelling"]); 119 | } 120 | 121 | #[test] 122 | /// ensure our calculations conform to the example provided at the link below 123 | /// 124 | /// https://www.kaggle.com/paulrohan2020/tf-idf-tutorial/notebook#TF-IDF-Model 125 | /// 126 | /// Consider a document containing 100 words wherein the word cat appears 3 times. 127 | /// The term frequency (i.e., tf) for cat is then (3 / 100) = 0.03. Now, assume we have 10 128 | /// million documents and the word cat appears in one thousand of these. Then, the inverse 129 | /// document frequency (i.e., idf) is calculated as log(10,000,000 / 1,000) = 4. Thus, the 130 | /// Tf-idf weight is the product of these quantities: 0.03 * 4 = 0.12. 131 | fn idf_returns_expected_value() { 132 | let num_docs = 10_000_000_f32; 133 | let num_occurrences = 1_000_f32; 134 | let abs_diff = (inverse_document_frequency(num_docs, num_occurrences) - 4.0).abs(); 135 | 136 | assert!(abs_diff <= f32::EPSILON); 137 | } 138 | 139 | #[test] 140 | /// ensure our calculations conform to the example provided at the link below 141 | /// 142 | /// https://www.kaggle.com/paulrohan2020/tf-idf-tutorial/notebook#TF-IDF-Model 143 | /// 144 | /// Consider a document containing 100 words wherein the word cat appears 3 times. 145 | /// The term frequency (i.e., tf) for cat is then (3 / 100) = 0.03. Now, assume we have 10 146 | /// million documents and the word cat appears in one thousand of these. Then, the inverse 147 | /// document frequency (i.e., idf) is calculated as log(10,000,000 / 1,000) = 4. Thus, the 148 | /// Tf-idf weight is the product of these quantities: 0.03 * 4 = 0.12. 149 | fn tf_idf_returns_expected_value() { 150 | let term_freq = 0.03_f32; 151 | let num_docs = 10_000_000_f32; 152 | let num_occurrences = 1_000_f32; 153 | let idf = inverse_document_frequency(num_docs, num_occurrences); 154 | let abs_diff = (tf_idf_score(term_freq, idf) - 0.12).abs(); 155 | 156 | assert!(abs_diff <= f32::EPSILON); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/progress.rs: -------------------------------------------------------------------------------- 1 | use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; 2 | use lazy_static::lazy_static; 3 | 4 | lazy_static! { 5 | /// Global progress bar that houses other progress bars 6 | pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout()); 7 | 8 | /// Global progress bar that is only used for printing messages that don't jack up other bars 9 | pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden); 10 | } 11 | 12 | /// Types of ProgressBars that can be added to `PROGRESS_BAR` 13 | #[derive(Copy, Clone)] 14 | pub enum BarType { 15 | /// no template used / not visible 16 | Hidden, 17 | 18 | /// normal directory status bar (reqs/sec shown) 19 | Default, 20 | 21 | /// similar to `Default`, except `-` is used in place of line/word/char count 22 | Message, 23 | 24 | /// bar used to show overall scan metrics 25 | Total, 26 | 27 | /// simpler output bar that shows only the directory being scanned (no updating info) 28 | Quiet, 29 | } 30 | 31 | /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html) 32 | /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html) 33 | pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar { 34 | let pb = ProgressBar::new(length).with_prefix(prefix.to_string()); 35 | 36 | update_style(&pb, bar_type); 37 | 38 | PROGRESS_BAR.add(pb) 39 | } 40 | 41 | /// Update the style of a progress bar based on the `BarType` 42 | pub fn update_style(bar: &ProgressBar, bar_type: BarType) { 43 | let mut style = ProgressStyle::default_bar().progress_chars("#>-").with_key( 44 | "smoothed_per_sec", 45 | |state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match ( 46 | state.pos(), 47 | state.elapsed().as_millis(), 48 | ) { 49 | // https://github.com/console-rs/indicatif/issues/394#issuecomment-1309971049 50 | // 51 | // indicatif released a change to how they reported eta/per_sec 52 | // and the results looked really weird based on how we use the progress 53 | // bars. this fixes that 54 | (pos, elapsed_ms) if elapsed_ms > 0 => { 55 | write!(w, "{:.0}/s", pos as f64 * 1000_f64 / elapsed_ms as f64).unwrap() 56 | } 57 | _ => write!(w, "-").unwrap(), 58 | }, 59 | ); 60 | 61 | style = match bar_type { 62 | BarType::Hidden => style.template("").unwrap(), 63 | BarType::Default => style 64 | .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {smoothed_per_sec:7} {prefix} {msg}") 65 | .unwrap(), 66 | BarType::Message => style 67 | .template(&format!( 68 | "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}} {{msg}}", 69 | "-" 70 | )) 71 | .unwrap(), 72 | BarType::Total => style 73 | .template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}") 74 | .unwrap(), 75 | BarType::Quiet => style.template("Scanning: {prefix}").unwrap(), 76 | }; 77 | 78 | bar.set_style(style); 79 | } 80 | 81 | #[cfg(test)] 82 | mod tests { 83 | use super::*; 84 | 85 | #[test] 86 | /// hit all code branches for add_bar 87 | fn add_bar_with_all_configurations() { 88 | let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden 89 | let p2 = add_bar("prefix", 2, BarType::Message); // no per second field 90 | let p3 = add_bar("prefix", 2, BarType::Default); // normal bar 91 | let p4 = add_bar("prefix", 2, BarType::Total); // totals bar 92 | 93 | p1.finish(); 94 | p2.finish(); 95 | p3.finish(); 96 | p4.finish(); 97 | 98 | assert!(p1.is_finished()); 99 | assert!(p2.is_finished()); 100 | assert!(p3.is_finished()); 101 | assert!(p4.is_finished()); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/scan_manager/mod.rs: -------------------------------------------------------------------------------- 1 | mod scan_container; 2 | mod response_container; 3 | mod scan; 4 | mod menu; 5 | mod utils; 6 | mod order; 7 | mod state; 8 | #[cfg(test)] 9 | mod tests; 10 | 11 | use menu::Menu; 12 | pub use menu::{MenuCmd, MenuCmdResult}; 13 | pub use order::ScanOrder; 14 | pub use response_container::FeroxResponses; 15 | pub use scan::{FeroxScan, ScanStatus, ScanType}; 16 | pub use scan_container::{FeroxScans, PAUSE_SCAN}; 17 | pub use state::FeroxState; 18 | pub use utils::{resume_scan, start_max_time_thread}; 19 | -------------------------------------------------------------------------------- /src/scan_manager/order.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Copy, Clone)] 2 | /// Simple enum to designate whether a URL was passed in by the user (Initial) or found during 3 | /// scanning (Latest) 4 | pub enum ScanOrder { 5 | /// Url was passed in by the user 6 | Initial, 7 | 8 | /// Url was found during scanning 9 | Latest, 10 | } 11 | -------------------------------------------------------------------------------- /src/scan_manager/response_container.rs: -------------------------------------------------------------------------------- 1 | use crate::response::FeroxResponse; 2 | use serde::{ser::SerializeSeq, Serialize, Serializer}; 3 | use std::sync::{Arc, RwLock}; 4 | 5 | /// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search 6 | #[derive(Debug, Default)] 7 | pub struct FeroxResponses { 8 | /// Internal structure: locked hashset of `FeroxScan`s 9 | pub responses: Arc>>, 10 | } 11 | 12 | /// Serialize implementation for FeroxResponses 13 | impl Serialize for FeroxResponses { 14 | /// Function that handles serialization of FeroxResponses 15 | fn serialize(&self, serializer: S) -> Result 16 | where 17 | S: Serializer, 18 | { 19 | if let Ok(responses) = self.responses.read() { 20 | let mut seq = serializer.serialize_seq(Some(responses.len()))?; 21 | 22 | for response in responses.iter() { 23 | seq.serialize_element(response)?; 24 | } 25 | 26 | seq.end() 27 | } else { 28 | // if for some reason we can't unlock the mutex, just write an empty list 29 | let seq = serializer.serialize_seq(Some(0))?; 30 | seq.end() 31 | } 32 | } 33 | } 34 | 35 | /// Implementation of `FeroxResponses` 36 | impl FeroxResponses { 37 | /// Add a `FeroxResponse` to the internal container 38 | pub fn insert(&self, response: FeroxResponse) { 39 | if let Ok(mut responses) = self.responses.write() { 40 | responses.push(response); 41 | } 42 | } 43 | 44 | /// Simple check for whether or not a FeroxResponse is contained within the inner container 45 | pub fn contains(&self, other: &FeroxResponse) -> bool { 46 | if let Ok(responses) = self.responses.read() { 47 | for response in responses.iter() { 48 | if response.url() == other.url() && response.method() == other.method() { 49 | return true; 50 | } 51 | } 52 | } 53 | false 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/scan_manager/state.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::filters::FeroxFilters; 3 | use crate::{config::Configuration, statistics::Stats, traits::FeroxSerialize, utils::fmt_err}; 4 | use anyhow::{Context, Result}; 5 | use serde::Serialize; 6 | use std::collections::HashSet; 7 | use std::sync::Arc; 8 | 9 | /// Data container for (de)?serialization of multiple items 10 | #[derive(Serialize, Debug)] 11 | pub struct FeroxState { 12 | /// Known scans 13 | scans: Arc, 14 | 15 | /// Current running config 16 | config: Arc, 17 | 18 | /// Known responses 19 | responses: &'static FeroxResponses, 20 | 21 | /// Gathered statistics 22 | statistics: Arc, 23 | 24 | /// collected extensions 25 | collected_extensions: HashSet, 26 | 27 | /// runtime filters, as they may differ from original config 28 | filters: Arc, 29 | } 30 | 31 | /// implementation of FeroxState 32 | impl FeroxState { 33 | /// create new FeroxState object 34 | pub fn new( 35 | scans: Arc, 36 | config: Arc, 37 | responses: &'static FeroxResponses, 38 | statistics: Arc, 39 | filters: Arc, 40 | ) -> Self { 41 | let collected_extensions = match scans.collected_extensions.read() { 42 | Ok(extensions) => extensions.clone(), 43 | Err(_) => HashSet::new(), 44 | }; 45 | 46 | Self { 47 | scans, 48 | config, 49 | responses, 50 | statistics, 51 | collected_extensions, 52 | filters, 53 | } 54 | } 55 | } 56 | 57 | /// FeroxSerialize implementation for FeroxState 58 | impl FeroxSerialize for FeroxState { 59 | /// Simply return debug format of FeroxState to satisfy as_str 60 | fn as_str(&self) -> String { 61 | format!("{self:?}") 62 | } 63 | 64 | /// Simple call to produce a JSON string using the given FeroxState 65 | fn as_json(&self) -> Result { 66 | serde_json::to_string(&self) 67 | .with_context(|| fmt_err("Could not convert scan's running state to JSON")) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/scan_manager/utils.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(test))] 2 | use crate::event_handlers::TermInputHandler; 3 | use crate::{ 4 | config::{Configuration, OutputLevel}, 5 | event_handlers::Handles, 6 | parser::TIMESPEC_REGEX, 7 | progress::BarType, 8 | scan_manager::scan::Visibility, 9 | scanner::RESPONSES, 10 | }; 11 | 12 | use std::{fs::File, io::BufReader, sync::Arc}; 13 | use tokio::time; 14 | 15 | /// Given a string representing some number of seconds, minutes, hours, or days, convert 16 | /// that representation to seconds and then wait for those seconds to elapse. Once that period 17 | /// of time has elapsed, kill all currently running scans and dump a state file to disk that can 18 | /// be used to resume any unfinished scan. 19 | pub async fn start_max_time_thread(handles: Arc) { 20 | log::trace!("enter: start_max_time_thread({:?})", handles); 21 | 22 | // as this function has already made it through the parser, which calls is_match on 23 | // the value passed to --time-limit using TIMESPEC_REGEX; we can safely assume that 24 | // the capture groups are populated; can expect something like 10m, 30s, 1h, etc... 25 | let captures = TIMESPEC_REGEX.captures(&handles.config.time_limit).unwrap(); 26 | let length_match = captures.get(1).unwrap(); 27 | let measurement_match = captures.get(2).unwrap(); 28 | 29 | if let Ok(length) = length_match.as_str().parse::() { 30 | let length_in_secs = match measurement_match.as_str().to_ascii_lowercase().as_str() { 31 | "s" => length, 32 | "m" => length * 60, // minutes 33 | "h" => length * 60 * 60, // hours 34 | "d" => length * 60 * 60 * 24, // days 35 | _ => length, 36 | }; 37 | 38 | log::debug!( 39 | "max time limit as string: {} and as seconds: {}", 40 | handles.config.time_limit, 41 | length_in_secs 42 | ); 43 | 44 | time::sleep(time::Duration::new(length_in_secs, 0)).await; 45 | 46 | log::trace!("exit: start_max_time_thread"); 47 | 48 | #[cfg(test)] 49 | panic!("{handles:?}"); 50 | #[cfg(not(test))] 51 | let _ = TermInputHandler::sigint_handler(handles.clone()); 52 | } 53 | 54 | log::warn!( 55 | "Could not parse the value provided ({}), can't enforce time limit", 56 | handles.config.time_limit 57 | ); 58 | } 59 | 60 | /// Primary logic used to load a Configuration from disk and populate the appropriate data 61 | /// structures 62 | pub fn resume_scan(filename: &str) -> Configuration { 63 | log::trace!("enter: resume_scan({})", filename); 64 | 65 | let file = File::open(filename).unwrap_or_else(|e| { 66 | log::error!("{}", e); 67 | log::error!("Could not open state file, exiting"); 68 | std::process::exit(1); 69 | }); 70 | 71 | let reader = BufReader::new(file); 72 | let state: serde_json::Value = serde_json::from_reader(reader).unwrap(); 73 | 74 | let conf = state.get("config").unwrap_or_else(|| { 75 | log::error!("Could not load configuration from state file, exiting"); 76 | std::process::exit(1); 77 | }); 78 | 79 | let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| { 80 | log::error!("{}", e); 81 | log::error!("Could not deserialize configuration found in state file, exiting"); 82 | std::process::exit(1); 83 | }); 84 | 85 | if let Some(responses) = state.get("responses") { 86 | if let Some(arr_responses) = responses.as_array() { 87 | for response in arr_responses { 88 | if let Ok(deser_resp) = serde_json::from_value(response.clone()) { 89 | RESPONSES.insert(deser_resp); 90 | } 91 | } 92 | } 93 | } 94 | 95 | log::trace!("exit: resume_scan -> {:?}", config); 96 | config 97 | } 98 | 99 | /// determine the type of progress bar to display 100 | /// takes both --limit-bars and output-level (--quiet|--silent|etc) 101 | /// into account to arrive at a `BarType` 102 | pub fn determine_bar_type( 103 | bar_limit: usize, 104 | number_of_bars: usize, 105 | output_level: OutputLevel, 106 | ) -> BarType { 107 | let visibility = if bar_limit == 0 { 108 | // no limit from cli, just set the value to visible 109 | // this protects us from a mutex unlock in number_of_bars 110 | // in the normal case 111 | Visibility::Visible 112 | } else if bar_limit < number_of_bars { 113 | // active bars exceed limit; hidden 114 | Visibility::Hidden 115 | } else { 116 | Visibility::Visible 117 | }; 118 | 119 | match (output_level, visibility) { 120 | (OutputLevel::Default, Visibility::Visible) => BarType::Default, 121 | (OutputLevel::Quiet, Visibility::Visible) => BarType::Quiet, 122 | (OutputLevel::Default, Visibility::Hidden) => BarType::Hidden, 123 | (OutputLevel::Quiet, Visibility::Hidden) => BarType::Hidden, 124 | (OutputLevel::Silent | OutputLevel::SilentJSON, _) => BarType::Hidden, 125 | } 126 | } 127 | 128 | #[cfg(test)] 129 | mod tests { 130 | use super::*; 131 | 132 | #[test] 133 | fn test_no_limit_visible() { 134 | let bar_type = determine_bar_type(0, 1, OutputLevel::Default); 135 | assert!(matches!(bar_type, BarType::Default)); 136 | } 137 | 138 | #[test] 139 | fn test_limit_exceeded_hidden() { 140 | let bar_type = determine_bar_type(1, 2, OutputLevel::Default); 141 | assert!(matches!(bar_type, BarType::Hidden)); 142 | } 143 | 144 | #[test] 145 | fn test_limit_not_exceeded_visible() { 146 | let bar_type = determine_bar_type(2, 1, OutputLevel::Default); 147 | assert!(matches!(bar_type, BarType::Default)); 148 | } 149 | 150 | #[test] 151 | fn test_quiet_visible() { 152 | let bar_type = determine_bar_type(0, 1, OutputLevel::Quiet); 153 | assert!(matches!(bar_type, BarType::Quiet)); 154 | } 155 | 156 | #[test] 157 | fn test_quiet_hidden() { 158 | let bar_type = determine_bar_type(1, 2, OutputLevel::Quiet); 159 | assert!(matches!(bar_type, BarType::Hidden)); 160 | } 161 | 162 | #[test] 163 | fn test_silent_hidden() { 164 | let bar_type = determine_bar_type(0, 1, OutputLevel::Silent); 165 | assert!(matches!(bar_type, BarType::Hidden)); 166 | } 167 | 168 | #[test] 169 | fn test_silent_json_hidden() { 170 | let bar_type = determine_bar_type(0, 1, OutputLevel::SilentJSON); 171 | assert!(matches!(bar_type, BarType::Hidden)); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/scanner/init.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | event_handlers::{Command::AddToUsizeField, Handles}, 3 | statistics::StatField::ExpectedPerScan, 4 | }; 5 | use anyhow::Result; 6 | use std::{convert::TryInto, sync::Arc}; 7 | 8 | /// Perform steps necessary to run scans that only need to be performed once (warming up the 9 | /// engine, as it were) 10 | pub async fn initialize(num_words: usize, handles: Arc) -> Result<()> { 11 | log::trace!("enter: initialize({}, {:?})", num_words, handles); 12 | 13 | // number of requests only needs to be calculated once, and then can be reused 14 | let num_reqs_expected: u64 = handles.expected_num_requests_per_dir().try_into()?; 15 | 16 | { 17 | // no real reason to keep the arc around beyond this call 18 | let scans = handles.ferox_scans()?; 19 | scans.set_bar_length(num_reqs_expected); 20 | } 21 | 22 | // tell Stats object about the number of expected requests 23 | handles 24 | .stats 25 | .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?; 26 | 27 | log::trace!("exit: initialize"); 28 | Ok(()) 29 | } 30 | -------------------------------------------------------------------------------- /src/scanner/limit_heap.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Debug, Formatter, Result}; 2 | 3 | /// bespoke variation on an array-backed max-heap 4 | /// 5 | /// 255 possible values generated from the initial requests/second 6 | /// 7 | /// when no additional errors are encountered, the left child is taken (increasing req/sec) 8 | /// if errors have increased since the last interval, the right child is taken (decreasing req/sec) 9 | /// 10 | /// formula for each child: 11 | /// - left: (|parent - current|) / 2 + current 12 | /// - right: current - ((|parent - current|) / 2) 13 | pub(super) struct LimitHeap { 14 | /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes ) 15 | pub(super) inner: [i32; 255], 16 | 17 | /// original # of requests / second 18 | pub(super) original: i32, 19 | 20 | /// current position w/in the backing array 21 | pub(super) current: usize, 22 | } 23 | 24 | /// default implementation of a LimitHeap 25 | impl Default for LimitHeap { 26 | /// zero-initialize the backing array 27 | fn default() -> Self { 28 | Self { 29 | inner: [0; 255], 30 | original: 0, 31 | current: 0, 32 | } 33 | } 34 | } 35 | 36 | /// Debug implementation of a LimitHeap 37 | impl Debug for LimitHeap { 38 | /// return debug representation that conforms to <32 elements in array 39 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 40 | let msg = format!( 41 | "LimitHeap {{ original: {}, current: {}, inner: [{}...] }}", 42 | self.original, self.current, self.inner[0] 43 | ); 44 | write!(f, "{msg}") 45 | } 46 | } 47 | 48 | /// implementation of a LimitHeap 49 | impl LimitHeap { 50 | /// move to right child, return node's index from which the move was requested 51 | pub(super) fn move_right(&mut self) -> usize { 52 | if self.has_children() { 53 | let tmp = self.current; 54 | self.current = self.current * 2 + 2; 55 | return tmp; 56 | } 57 | self.current 58 | } 59 | 60 | /// move to left child, return node's index from which the move was requested 61 | pub(super) fn move_left(&mut self) -> usize { 62 | if self.has_children() { 63 | let tmp = self.current; 64 | self.current = self.current * 2 + 1; 65 | return tmp; 66 | } 67 | self.current 68 | } 69 | 70 | /// move to parent, return node's index from which the move was requested 71 | pub(super) fn move_up(&mut self) -> usize { 72 | if self.has_parent() { 73 | let tmp = self.current; 74 | self.current = (self.current - 1) / 2; 75 | return tmp; 76 | } 77 | self.current 78 | } 79 | 80 | /// move directly to the given index 81 | pub(super) fn move_to(&mut self, index: usize) { 82 | self.current = index; 83 | } 84 | 85 | /// get the current node's value 86 | pub(super) fn value(&self) -> i32 { 87 | self.inner[self.current] 88 | } 89 | 90 | /// set the current node's value 91 | pub(super) fn set_value(&mut self, value: i32) { 92 | self.inner[self.current] = value; 93 | } 94 | 95 | /// check that this node has a parent (true for all except root) 96 | pub(super) fn has_parent(&self) -> bool { 97 | self.current > 0 98 | } 99 | 100 | /// get node's parent's value or self.original if at the root 101 | pub(super) fn parent_value(&mut self) -> i32 { 102 | if self.has_parent() { 103 | let current = self.move_up(); 104 | let val = self.value(); 105 | self.move_to(current); 106 | return val; 107 | } 108 | self.original 109 | } 110 | 111 | /// check if the current node has children 112 | pub(super) fn has_children(&self) -> bool { 113 | // inner structure is a complete tree, just check for the right child 114 | self.current * 2 + 2 <= self.inner.len() 115 | } 116 | 117 | /// get current node's right child's value 118 | fn right_child_value(&mut self) -> i32 { 119 | let tmp = self.move_right(); 120 | let val = self.value(); 121 | self.move_to(tmp); 122 | val 123 | } 124 | 125 | /// set current node's left child's value 126 | fn set_left_child(&mut self) { 127 | let parent = self.parent_value(); 128 | let current = self.value(); 129 | let value = ((parent - current).abs() / 2) + current; 130 | 131 | self.move_left(); 132 | self.set_value(value); 133 | self.move_up(); 134 | } 135 | 136 | /// set current node's right child's value 137 | fn set_right_child(&mut self) { 138 | let parent = self.parent_value(); 139 | let current = self.value(); 140 | let value = current - ((parent - current).abs() / 2); 141 | 142 | self.move_right(); 143 | self.set_value(value); 144 | self.move_up(); 145 | } 146 | 147 | /// iterate over the backing array, filling in each child's value based on the original value 148 | pub(super) fn build(&mut self) { 149 | // ex: original is 400 150 | // arr[0] == 200 151 | // arr[1] (left child) == 300 152 | // arr[2] (right child) == 100 153 | let root = self.original / 2; 154 | 155 | self.inner[0] = root; // set root node to half of the original value 156 | self.inner[1] = ((self.original - root).abs() / 2) + root; 157 | self.inner[2] = root - ((self.original - root).abs() / 2); 158 | 159 | // start with index 1 and fill in each child below that node 160 | for i in 1..self.inner.len() { 161 | self.move_to(i); 162 | 163 | if self.has_children() && self.right_child_value() == 0 { 164 | // this node has an unset child since the rchild is 0 165 | self.set_left_child(); 166 | self.set_right_child(); 167 | } 168 | } 169 | self.move_to(0); // reset current index to the root of the tree 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /src/scanner/mod.rs: -------------------------------------------------------------------------------- 1 | mod ferox_scanner; 2 | mod utils; 3 | mod init; 4 | #[cfg(test)] 5 | mod tests; 6 | mod limit_heap; 7 | mod policy_data; 8 | mod requester; 9 | 10 | pub use self::ferox_scanner::{FeroxScanner, RESPONSES}; 11 | pub use self::init::initialize; 12 | pub use self::utils::PolicyTrigger; 13 | -------------------------------------------------------------------------------- /src/scanner/tests.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use tokio::sync::Semaphore; 4 | 5 | use crate::{ 6 | config::OutputLevel, 7 | event_handlers::Handles, 8 | scan_manager::{FeroxScans, ScanOrder}, 9 | }; 10 | 11 | use super::*; 12 | 13 | #[tokio::test(flavor = "multi_thread", worker_threads = 1)] 14 | #[should_panic] 15 | /// try to hit struct field coverage of FileOutHandler 16 | async fn get_scan_by_url_bails_on_unfound_url() { 17 | let sem = Semaphore::new(10); 18 | let urls = FeroxScans::new(OutputLevel::Default, 0); 19 | 20 | let scanner = FeroxScanner::new( 21 | "http://localhost", 22 | ScanOrder::Initial, 23 | Arc::new(Default::default()), 24 | Arc::new(sem), 25 | Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0), 26 | ); 27 | scanner.scan_url().await.unwrap(); 28 | } 29 | -------------------------------------------------------------------------------- /src/scanner/utils.rs: -------------------------------------------------------------------------------- 1 | #[derive(Copy, Clone, PartialEq, Eq, Debug)] 2 | /// represents different situations where different criteria can trigger auto-tune/bail behavior 3 | pub enum PolicyTrigger { 4 | /// excessive 403 trigger 5 | Status403, 6 | 7 | /// excessive 429 trigger 8 | Status429, 9 | 10 | /// excessive general errors 11 | Errors, 12 | 13 | /// dummy error for upward rate adjustment 14 | TryAdjustUp, 15 | } 16 | -------------------------------------------------------------------------------- /src/statistics/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Copy, Clone)] 2 | /// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated 3 | pub enum StatError { 4 | /// Represents a timeout error 5 | Timeout, 6 | 7 | /// Represents a URL formatting error 8 | UrlFormat, 9 | 10 | /// Represents an error encountered during redirection 11 | Redirection, 12 | 13 | /// Represents an error encountered during connection 14 | Connection, 15 | 16 | /// Represents an error resulting from the client's request 17 | Request, 18 | 19 | /// Represents any other error not explicitly defined above 20 | Other, 21 | } 22 | -------------------------------------------------------------------------------- /src/statistics/field.rs: -------------------------------------------------------------------------------- 1 | /// Enum representing fields whose updates need to be performed in batches instead of one at 2 | /// a time 3 | #[derive(Debug, Copy, Clone)] 4 | pub enum StatField { 5 | /// Due to the necessary order of events, the number of requests expected to be sent isn't 6 | /// known until after `statistics::initialize` is called. This command allows for updating 7 | /// the `expected_per_scan` field after initialization 8 | ExpectedPerScan, 9 | 10 | /// Translates to `total_scans` 11 | TotalScans, 12 | 13 | /// Translates to `links_extracted` 14 | LinksExtracted, 15 | 16 | /// Translates to `extensions_collected` 17 | ExtensionsCollected, 18 | 19 | /// Translates to `total_expected` 20 | TotalExpected, 21 | 22 | /// Translates to `wildcards_filtered` 23 | WildcardsFiltered, 24 | 25 | /// Translates to `responses_filtered` 26 | ResponsesFiltered, 27 | 28 | /// Translates to `resources_discovered` 29 | ResourcesDiscovered, 30 | 31 | /// Translates to `initial_targets` 32 | InitialTargets, 33 | 34 | /// Translates to `directory_scan_times`; assumes a single append to the vector 35 | DirScanTimes, 36 | } 37 | -------------------------------------------------------------------------------- /src/statistics/init.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/statistics/macros.rs: -------------------------------------------------------------------------------- 1 | #![macro_use] 2 | 3 | /// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times 4 | /// 5 | /// default is to increment by 1, second arg can be used to increment by a different value 6 | #[macro_export] 7 | macro_rules! atomic_increment { 8 | ($metric:expr) => { 9 | $metric.fetch_add(1, Ordering::Relaxed); 10 | }; 11 | 12 | ($metric:expr, $value:expr) => { 13 | $metric.fetch_add($value, Ordering::Relaxed); 14 | }; 15 | } 16 | 17 | /// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times 18 | #[macro_export] 19 | macro_rules! atomic_load { 20 | ($metric:expr) => { 21 | $metric.load(Ordering::Relaxed) 22 | }; 23 | ($metric:expr, $ordering:expr) => { 24 | $metric.load($ordering) 25 | }; 26 | } 27 | 28 | /// Wrapper around `Atomic*.store` to save me from writing Ordering::Relaxed a bajillion times 29 | #[macro_export] 30 | macro_rules! atomic_store { 31 | ($metric:expr, $value:expr) => { 32 | $metric.store($value, Ordering::Relaxed); 33 | }; 34 | ($metric:expr, $value:expr, $ordering:expr) => { 35 | $metric.store($value, $ordering); 36 | }; 37 | } 38 | -------------------------------------------------------------------------------- /src/statistics/mod.rs: -------------------------------------------------------------------------------- 1 | mod error; 2 | mod macros; 3 | mod container; 4 | mod field; 5 | #[cfg(test)] 6 | mod tests; 7 | 8 | pub use self::container::Stats; 9 | pub use self::error::StatError; 10 | pub use self::field::StatField; 11 | 12 | #[cfg(test)] 13 | use self::tests::{setup_stats_test, teardown_stats_test}; 14 | -------------------------------------------------------------------------------- /src/statistics/tests.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | use crate::{ 3 | config::Configuration, 4 | event_handlers::{Command, StatsHandle, StatsHandler}, 5 | traits::FeroxSerialize, 6 | CommandSender, Joiner, 7 | }; 8 | use anyhow::Result; 9 | use reqwest::StatusCode; 10 | use std::sync::Arc; 11 | use tempfile::NamedTempFile; 12 | 13 | /// simple helper to reduce code reuse 14 | pub fn setup_stats_test() -> (Joiner, StatsHandle) { 15 | let config = Arc::new(Configuration::new().unwrap()); 16 | StatsHandler::initialize(config) 17 | } 18 | 19 | /// another helper to stay DRY; must be called after any sent commands and before any checks 20 | /// performed against the Stats object 21 | pub async fn teardown_stats_test(sender: CommandSender, task: Joiner) { 22 | // send exit and await, once the await completes, stats should be updated 23 | sender.send(Command::Exit).unwrap_or_default(); 24 | task.await.unwrap().unwrap(); 25 | } 26 | 27 | #[tokio::test(flavor = "multi_thread", worker_threads = 1)] 28 | /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise) 29 | async fn statistics_handler_exits() -> Result<()> { 30 | let (task, handle) = setup_stats_test(); 31 | 32 | handle.tx.send(Command::Exit)?; 33 | 34 | task.await??; // blocks on the handler's while loop 35 | 36 | // if we've made it here, the test has succeeded 37 | Ok(()) 38 | } 39 | 40 | #[test] 41 | /// Stats::save should write contents of Stats to disk 42 | fn save_writes_stats_object_to_disk() { 43 | let config = Configuration::new().unwrap(); 44 | let stats = Stats::new(config.json); 45 | 46 | stats.add_request(); 47 | stats.add_request(); 48 | stats.add_request(); 49 | stats.add_request(); 50 | stats.add_error(StatError::Timeout); 51 | stats.add_error(StatError::Timeout); 52 | stats.add_error(StatError::Timeout); 53 | stats.add_error(StatError::Timeout); 54 | stats.add_status_code(StatusCode::OK); 55 | stats.add_status_code(StatusCode::OK); 56 | stats.add_status_code(StatusCode::OK); 57 | let outfile = NamedTempFile::new().unwrap(); 58 | assert!(stats.save(174.33, outfile.path().to_str().unwrap()).is_ok()); 59 | 60 | assert!(stats.as_json().unwrap().contains("statistics")); 61 | assert!(stats.as_json().unwrap().contains("11")); // requests made 62 | assert!(stats.as_str().is_empty()); 63 | } 64 | -------------------------------------------------------------------------------- /src/traits.rs: -------------------------------------------------------------------------------- 1 | //! collection of all traits used 2 | use crate::filters::{ 3 | LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WildcardFilter, 4 | WordsFilter, 5 | }; 6 | use crate::response::FeroxResponse; 7 | use crate::utils::status_colorizer; 8 | use anyhow::Result; 9 | use crossterm::style::{style, Stylize}; 10 | use serde::Serialize; 11 | use std::any::Any; 12 | use std::fmt::{self, Debug, Display, Formatter}; 13 | 14 | // references: 15 | // https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5 16 | // https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects 17 | 18 | /// FeroxFilter trait; represents different types of possible filters that can be applied to 19 | /// responses 20 | pub trait FeroxFilter: Debug + Send + Sync { 21 | /// Determine whether or not this particular filter should be applied or not 22 | fn should_filter_response(&self, response: &FeroxResponse) -> bool; 23 | 24 | /// delegates to the FeroxFilter-implementing type which gives us the actual type of self 25 | fn box_eq(&self, other: &dyn Any) -> bool; 26 | 27 | /// gives us `other` as Any in box_eq 28 | fn as_any(&self) -> &dyn Any; 29 | } 30 | 31 | impl Display for dyn FeroxFilter { 32 | fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> { 33 | if let Some(filter) = self.as_any().downcast_ref::() { 34 | write!(f, "Line count: {}", style(filter.line_count).cyan()) 35 | } else if let Some(filter) = self.as_any().downcast_ref::() { 36 | write!(f, "Word count: {}", style(filter.word_count).cyan()) 37 | } else if let Some(filter) = self.as_any().downcast_ref::() { 38 | write!(f, "Response size: {}", style(filter.content_length).cyan()) 39 | } else if let Some(filter) = self.as_any().downcast_ref::() { 40 | write!(f, "Regex: {}", style(&filter.raw_string).cyan()) 41 | } else if let Some(filter) = self.as_any().downcast_ref::() { 42 | let mut msg = format!( 43 | "{} requests with {} responses ", 44 | style(&filter.method).cyan(), 45 | status_colorizer(&filter.status_code.to_string()) 46 | ); 47 | 48 | match (filter.content_length, filter.word_count, filter.line_count) { 49 | (None, None, None) => { 50 | unreachable!("wildcard filter without any filters set"); 51 | } 52 | (None, None, Some(lc)) => { 53 | msg.push_str(&format!("containing {} lines", lc)); 54 | } 55 | (None, Some(wc), None) => { 56 | msg.push_str(&format!("containing {} words", wc)); 57 | } 58 | (None, Some(wc), Some(lc)) => { 59 | msg.push_str(&format!("containing {} words and {} lines", wc, lc)); 60 | } 61 | (Some(cl), None, None) => { 62 | msg.push_str(&format!("containing {} bytes", cl)); 63 | } 64 | (Some(cl), None, Some(lc)) => { 65 | msg.push_str(&format!("containing {} bytes and {} lines", cl, lc)); 66 | } 67 | (Some(cl), Some(wc), None) => { 68 | msg.push_str(&format!("containing {} bytes and {} words", cl, wc)); 69 | } 70 | (Some(cl), Some(wc), Some(lc)) => { 71 | msg.push_str(&format!( 72 | "containing {} bytes, {} words, and {} lines", 73 | cl, wc, lc 74 | )); 75 | } 76 | } 77 | 78 | write!(f, "{}", msg) 79 | } else if let Some(filter) = self.as_any().downcast_ref::() { 80 | write!(f, "Status code: {}", style(filter.filter_code).cyan()) 81 | } else if let Some(filter) = self.as_any().downcast_ref::() { 82 | write!( 83 | f, 84 | "Pages similar to: {}", 85 | style(&filter.original_url).cyan() 86 | ) 87 | } else { 88 | write!(f, "Filter: {self:?}") 89 | } 90 | } 91 | } 92 | 93 | /// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object" 94 | /// error when attempting to derive PartialEq on the trait itself 95 | impl PartialEq for Box { 96 | /// Perform a comparison of two implementors of the FeroxFilter trait 97 | fn eq(&self, other: &Box) -> bool { 98 | self.box_eq(other.as_any()) 99 | } 100 | } 101 | 102 | /// FeroxSerialize trait; represents different types that are Serialize and also implement 103 | /// as_str / as_json methods 104 | pub trait FeroxSerialize: Serialize { 105 | /// Return a String representation of the object, generally the human readable version of the 106 | /// implementor 107 | fn as_str(&self) -> String; 108 | 109 | /// Return an NDJSON representation of the object 110 | fn as_json(&self) -> Result; 111 | } 112 | -------------------------------------------------------------------------------- /tests/mutual-auth/Caddyfile: -------------------------------------------------------------------------------- 1 | (mTLS) { 2 | tls { 3 | client_auth { 4 | mode require_and_verify 5 | trusted_ca_cert_file certs/server/ca.crt 6 | } 7 | } 8 | } 9 | 10 | https://localhost:8001 { 11 | import mTLS 12 | log 13 | 14 | handle / { 15 | file_server browse 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/mutual-auth/README.md: -------------------------------------------------------------------------------- 1 | # Testing mTLS 2 | 3 | - run `gen-certs.sh` 4 | - run `sudo /path/to/caddy run` 5 | - expect listener on port 8001 6 | - run `feroxbuster -u https://localhost:8001 --client-key certs/client/client.key --client-cert certs/client/client.crt` 7 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/client/client.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICqzCCAZMCFE22XDzrLwkJIkb3EdP333d4HoXQMA0GCSqGSIb3DQEBCwUAMBMx 3 | ETAPBgNVBAMMCFNlcnZlckNBMB4XDTIzMDUwNjExMDYyM1oXDTI0MDUwNTExMDYy 4 | M1owETEPMA0GA1UEAwwGQ2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB 5 | CgKCAQEAz3EPWMsh+dfPdbHtpNhizZZs+r0djzdHHgkbnNQ1PodWDnv0Rf1YgNEa 6 | umQuUvIgjMtorRqbz9HLG4+H2aR5KHgPwBNHyKS4PEiQvWDV88aJxdMbgL/IfzAt 7 | di85UcBUkyqUe1r6vIS0smJo1wVwxLEmD6kdt1BEI3LaK1j99JeG8TAS8f+/xf4s 8 | ouE4lA+y3bJQP18wUGuyudntFQBKgjY2Tx+RWbBcx0zW68M7IMQ5bDz0oK9MYw8G 9 | q2vwcRyMLuoyNpbDT5mI2wsQu/r2O0CCNbtkg5JxasdYR7Llw9YTl74st3dshM9e 10 | 4V5uuVotcWXW6U518nWHOQy9qiBSOQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB4 11 | xOVvWrRZ4SBqzaen32COXpjddX28Q7YmNB/UKl3ZT7R1dIjUMfJz2le0mj2UpSAr 12 | rDT7PCsXnDP0KswGiJC3IVTa/hnkUk798jwUvp221jvebyy8/NMWfWPoIKfhELdb 13 | 3uJfrGyQuB8Zf9Q1hc9jYDX27EbGaDSpOrpE9Ej2riVnbgBKZsS5jcfY8JDrkv+F 14 | 4cP2pTu6mVRuU1Bzx3SB0Vg2uGi1QTJuuA905Y3zpoRfTtybKlRRkMQk+46xrdyV 15 | x64wq9zcL6Kq4D/UE3EjLnjbRw6H6g8jbnBjT5KRfP2tmbF9RTZs44Dl0hYvXber 16 | HrvWtxHG8OJ8BLQg1rQd 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/client/client.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDPcQ9YyyH51891 3 | se2k2GLNlmz6vR2PN0ceCRuc1DU+h1YOe/RF/ViA0Rq6ZC5S8iCMy2itGpvP0csb 4 | j4fZpHkoeA/AE0fIpLg8SJC9YNXzxonF0xuAv8h/MC12LzlRwFSTKpR7Wvq8hLSy 5 | YmjXBXDEsSYPqR23UEQjctorWP30l4bxMBLx/7/F/iyi4TiUD7LdslA/XzBQa7K5 6 | 2e0VAEqCNjZPH5FZsFzHTNbrwzsgxDlsPPSgr0xjDwara/BxHIwu6jI2lsNPmYjb 7 | CxC7+vY7QII1u2SDknFqx1hHsuXD1hOXviy3d2yEz17hXm65Wi1xZdbpTnXydYc5 8 | DL2qIFI5AgMBAAECggEAC8XVeoM1w4uITDxLucMnkVYgC3dj5/K5zCY1bVg8SNcO 9 | rt4BSh8TkKT9ZLZmjCHOb9sj7s4PqXLVOXRTAAq17xJoR2z4shYKGC7AmyTVo6MB 10 | AuuFGDCaMQCzlc1ejgmRqzP7jwgl6oDIDgcofsqB4MHSgIlHJNYO9emQ4OypJgJA 11 | xd8KT5S/hThJG1VqJ6P0oiB/WBlzcJ5wX4GSVE25RlpRX8ogqCyI9V+SRq2CrG7U 12 | Jqv3Kbag7derTfqmsKyjv/kckOgfKH/rm61HMrYshcPfgxL2fZe2Q8wCTexvhZwZ 13 | 8vD8bvR++SxOxbigCIB7ReYgmoj4bocjqDX4vUhe8QKBgQD35oDdOa2uiOs7NWVf 14 | IV1ZwPWxxwnYFIEA8paQwsYGIxHrYNdGSsGBzwvLDPpTeOO0VdoC+sP5zytTv547 15 | djeOzGf9Hj6swa5tPdzkYjZV/85mnmGKaEmmCN4AvpYol5l2BTetFtX0v6QEaqvU 16 | uZbV5X2UcuClExA0frNUJDVHkQKBgQDWOCZq1r9X3iEkcFSBhironVNj80jFqIum 17 | rMbGUUcOI05U2hkmMDluSW1NNL2k+SNJXq7fmkjIQEXffqcbsXUSIQB0MU6yddt5 18 | 7+c19ioZChx91Kl049rKQ21kPTh7D0TCUvDQLapt2xbUNg6rGCLSrkkVlWwxLnDU 19 | pNk/c4QcKQKBgEreedLWhabtwSV7pecKO5hM16dedpGk96UinuiPeqEF3HabI8kd 20 | 8L1Um7oybDPjkdm4CATYWXHL6Mj9WTuaI4NkJo/in4krYZOqmFj9dG2auWpysQDN 21 | KFkV2n6dENqnlnh3cO48tFebvVx8HvM7Ldvh2ICKBWC1ljJUhbKG0PSRAoGBAJVy 22 | fNLCWKEbVbHPMBVgnaTExT2Qp29F4493MAGBCHpDhU1LDoqG0DoxvbBEIB3stYJl 23 | LMjQIQCbXmPKPxjh15O7NE7ba1SzRleuV3Zc8wee9zuN1l6265d6LOHml/W6NDUB 24 | mgESKrkTRLztrZQNdZXXgyMsqFszVAH1s55Bn6PpAoGBAN13Ev7Ynysdvkc3aHO6 25 | qM0hH6mAlEOAyCTk5r/0cyz9rGyYWXiVXen0ftSaBcISdzhrVkRDs3rLrHwEXdu1 26 | Y2Z1HhZkILw/C4t+Eaa6FOWfwwPAdOpaxYpxKxCEeCBKmkd1z0Dx0vDEDrt+AaHa 27 | UYIQ9wAbZpuKGfFQceyr1lBO 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/server/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDBzCCAe+gAwIBAgIUG3vb4pIbvaI/+LzOpu6Z4b6s4iIwDQYJKoZIhvcNAQEL 3 | BQAwEzERMA8GA1UEAwwIU2VydmVyQ0EwHhcNMjMwNTA2MTEwNjIzWhcNMzMwNTAz 4 | MTEwNjIzWjATMREwDwYDVQQDDAhTZXJ2ZXJDQTCCASIwDQYJKoZIhvcNAQEBBQAD 5 | ggEPADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS 6 | /hyBvAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARq 7 | WwDhpa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ 8 | 9Zn9ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+m 9 | MBqrK3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iH 10 | EIMp9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAAaNTMFEwHQYDVR0OBBYE 11 | FLlRKsDb/ducVIBirME0VJZ3TwfkMB8GA1UdIwQYMBaAFLlRKsDb/ducVIBirME0 12 | VJZ3TwfkMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAD7lqNzU 13 | wxuyO60Gn2q6DUBb1Kseq6bSndNHeagdfMfKManKl1YObnB0ciTO3bnmNXiXktSu 14 | BsQzlmr3O+H6X39Vpdyqq4SoOcOt0I+bvBykk1UZqEoc7jGXdZVmnk9Q0uoKtWxJ 15 | rV9CHEhyPNnEh4W07y05UUn9S6EiKy5232yi4USdmk44GXhFblS5inhTTxca2vEq 16 | 9h+FH+QZ7ehaAaWR+EaQjXNwm2mN7gWxM3Q6RfK9N67MHD9ggmfdyZmnyt5gCidC 17 | ys4W4stEh6d6fXZT77dcGaHKdXW3GwP3ZcAlRFYPqpAvWzndC9kDCgIULeSP1ALy 18 | cILcb0HQvNS0t60= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/server/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICrzCCAZcCFGMKRtmMLuut+sxC+TbWQfum7oXZMA0GCSqGSIb3DQEBCwUAMBQx 3 | EjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMzA1MDYxMTA2MjNaFw0zMzA1MDMxMTA2 4 | MjNaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP 5 | ADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS/hyB 6 | vAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARqWwDh 7 | pa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ9Zn9 8 | ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+mMBqr 9 | K3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iHEIMp 10 | 9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAATANBgkqhkiG9w0BAQsFAAOC 11 | AQEAjPAtZs1by2h/1fr/ypojw16llzbReT8J+T8YHSTf6YwjoE83I0QDOLEo1ax+ 12 | e/8qyQLs0EnlfdomNyA4Z/ECbY5c1nY0Dp//u6WH7AwLUx5HiwUw4Fmxu9Q/oB1o 13 | 3vhIPl5Vd/VpdxDzuO8q8WvagwjVaxsZP3PVaBDRzZZPldPgTakfk+w5XnjNfgJi 14 | RDRutTRe6KBOxt7PAzAVV71FtOIq0b4xCNJGNurYBhRgZ5iQ7yMw+I5Vte1TakWr 15 | 9gfE/yoKbU1W+y0QxSDTsnTCO4i3mXmBTuceTVWELwqZcr34W7n3vD8UtZQfanML 16 | cHCZaLPSMDuDtS74FSamP3i+oQ== 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/server/server.crt.1: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICrzCCAZcCFGMKRtmMLuut+sxC+TbWQfum7oXZMA0GCSqGSIb3DQEBCwUAMBQx 3 | EjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMzA1MDYxMTA2MjNaFw0zMzA1MDMxMTA2 4 | MjNaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP 5 | ADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS/hyB 6 | vAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARqWwDh 7 | pa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ9Zn9 8 | ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+mMBqr 9 | K3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iHEIMp 10 | 9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAATANBgkqhkiG9w0BAQsFAAOC 11 | AQEAjPAtZs1by2h/1fr/ypojw16llzbReT8J+T8YHSTf6YwjoE83I0QDOLEo1ax+ 12 | e/8qyQLs0EnlfdomNyA4Z/ECbY5c1nY0Dp//u6WH7AwLUx5HiwUw4Fmxu9Q/oB1o 13 | 3vhIPl5Vd/VpdxDzuO8q8WvagwjVaxsZP3PVaBDRzZZPldPgTakfk+w5XnjNfgJi 14 | RDRutTRe6KBOxt7PAzAVV71FtOIq0b4xCNJGNurYBhRgZ5iQ7yMw+I5Vte1TakWr 15 | 9gfE/yoKbU1W+y0QxSDTsnTCO4i3mXmBTuceTVWELwqZcr34W7n3vD8UtZQfanML 16 | cHCZaLPSMDuDtS74FSamP3i+oQ== 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/server/server.crt.2: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICrzCCAZcCFGMKRtmMLuut+sxC+TbWQfum7oXZMA0GCSqGSIb3DQEBCwUAMBQx 3 | EjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMzA1MDYxMTA2MjNaFw0zMzA1MDMxMTA2 4 | MjNaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP 5 | ADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS/hyB 6 | vAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARqWwDh 7 | pa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ9Zn9 8 | ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+mMBqr 9 | K3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iHEIMp 10 | 9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAATANBgkqhkiG9w0BAQsFAAOC 11 | AQEAjPAtZs1by2h/1fr/ypojw16llzbReT8J+T8YHSTf6YwjoE83I0QDOLEo1ax+ 12 | e/8qyQLs0EnlfdomNyA4Z/ECbY5c1nY0Dp//u6WH7AwLUx5HiwUw4Fmxu9Q/oB1o 13 | 3vhIPl5Vd/VpdxDzuO8q8WvagwjVaxsZP3PVaBDRzZZPldPgTakfk+w5XnjNfgJi 14 | RDRutTRe6KBOxt7PAzAVV71FtOIq0b4xCNJGNurYBhRgZ5iQ7yMw+I5Vte1TakWr 15 | 9gfE/yoKbU1W+y0QxSDTsnTCO4i3mXmBTuceTVWELwqZcr34W7n3vD8UtZQfanML 16 | cHCZaLPSMDuDtS74FSamP3i+oQ== 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /tests/mutual-auth/certs/server/server.der: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/epi052/feroxbuster/e321a4e0e6b114787aa5674e16732e1a4482bbd4/tests/mutual-auth/certs/server/server.der -------------------------------------------------------------------------------- /tests/mutual-auth/certs/server/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmszSj2h/Cw83G 3 | mUbwS6wzSZC9abpJgd+pm3M49O4WVWz5kv4cgbwHQRaJtGU8OXSKbFLUUmWwz67A 4 | laXWPTtY8cIaw4upn78ibSixG01uD7wEalsA4aWtkmMATCPtIpV1kwM8Cblk9TYb 5 | lLAHAekhsJDZt4iqmVA8Z91di9gB0XChCfWZ/Xjz+AbL7QEvGQS8U56KO0dkh4ls 6 | Iw8Lk7UuH0OW1sxpJnZnrieySbeGc0FfpjAaqyt11YvjF2sr7eWn3xrx4/yroZVQ 7 | Vx51IErxIBaboSvXpbHbNokYaPJqdXdIhxCDKfSx1X7bCKVMWyrfz2vyS+6BvSwS 8 | Lz4/0iI7AgMBAAECggEAKvz2u7Rh0WOWGrtnQEt7bkRv13C+8frUd1QXnB4JkefY 9 | sOmXrzlDiGlgCwXiv2ufopy5pXhUMgr0qUROHlfvCIpbwHQh/Y2tCA83WajNSG81 10 | ULwumKUYCRFBh4+bCimLemT9hguJ7D+SAv3OgRgciywRxpteWoQr3U/5lYidHSZ/ 11 | gv13lVKbn72zD5opeGA2hS1MlLZV/xueSvhT3lzbv61hqdersACj2Tvi8O2/imVy 12 | XjEZnPRQhlFPtiAN5J3on6Xo+MqieuhA3yxhBrYoLsMrTCK6ePThdXfcgmpEjQ8l 13 | 6HxNmnPri5KbxCTbGgCjMiSidnRim2IpBMEP32eN8QKBgQDLr2CoMdyliFU6Gm1P 14 | rxWTMnvzdVbXUp4B8YEdyNyKdWt50cqbB3UvnFX2gpELYdy3uYcXTKm+Nynruam1 15 | Z/Ya1HXwN+wdgQhjq9n4izLvEfUkXWDNNikQmts8Uxkp+uEK+OOp1/NjZlA6YdS3 16 | crq5wPxLoAP2JxiaoIJF74QMqwKBgQDRhABgZbWVHwPcLVVqZ5+MJYvQORqIqapf 17 | kGe/jR/CMC0Tkop2O1tY3f68bMNKkXfj7QEtDlppMswZ9MOqBBr56yGZzrQa+cB3 18 | lF4+hP06OvyIkdmZlP4NHm/DtF9gt1KjWPF2VcIfD3VfZO8E/XJn2n1KnKCU+4cb 19 | lyJYi9AgsQKBgQCWkgPy8kE5QSo3tJeAI17gnJ5SoDhdHp7dsukO2pBl7l1QBY0v 20 | w3iWhIxrmaOddW+ThZve1nZYvjDIKEzTZJHizZKNzNlICj3oaH7OpCA36N9+TWUk 21 | 7le3BbLxykA870/zK4Ao6xHqNhUyw2VbY32zmX0obpbfHZGrpOIIzwGf1wKBgDlM 22 | U1oJls5QbBrT3w85hZ2rSwBIDaSgWfLGqEjvjGbsC/fVVL6e3w1/sMHRMNt8yv/v 23 | einbSgiJFt5mXPhrJQGCN28742+ZK/TIA7ovXp2FMjkbQhpJb+0gjMpF0uu9VwFL 24 | OsX1ECC0dpH/JYsE0TvrueYkzZnQ7BM0kvUKT4IRAoGAOPVed0zkDh3iobQ3A3IG 25 | JepRygabC68iOHlrD6sVxST0HdyP9pxwMe9gnz5TDAZkWvhJV0UUmaMCpbShsc+n 26 | ymKSNnXAxt+G6XHH3Mg9aDNi70og4HhhT6dU2579xUOBY2057ZgpWXK3rf+JKls4 27 | XlkplyHw0UqkEhCw+FMa3Gs= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /tests/mutual-auth/gen-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Create server and client certificate directories 4 | mkdir -p certs/server 5 | mkdir -p certs/client 6 | 7 | # Generate server key 8 | openssl genrsa -out certs/server/server.key 2048 9 | 10 | # Generate a Certificate Signing Request (CSR) for the server key 11 | openssl req -new -key certs/server/server.key -out certs/server/server.csr -subj "/CN=localhost" 12 | 13 | # Self-sign the server CSR to create the server certificate 14 | openssl x509 -req -in certs/server/server.csr -signkey certs/server/server.key -out certs/server/server.crt -days 3650 15 | 16 | # Generate server-side Certificate Authority (CA) file 17 | openssl req -x509 -nodes -new -key certs/server/server.key -sha256 -days 3650 -out certs/server/ca.crt -subj "/CN=ServerCA" 18 | 19 | # Generate client key 20 | openssl genrsa -out certs/client/client.key 2048 21 | 22 | # Generate a Certificate Signing Request (CSR) for the client key 23 | openssl req -new -key certs/client/client.key -out certs/client/client.csr -subj "/CN=Client" 24 | 25 | # Sign the client CSR with the server CA to create the client certificate 26 | openssl x509 -req -in certs/client/client.csr -CA certs/server/ca.crt -CAkey certs/server/server.key -CAcreateserial -out certs/client/client.crt -days 365 27 | 28 | # Cleanup 29 | rm -f certs/server/server.csr 30 | rm -f certs/client/client.csr 31 | 32 | -------------------------------------------------------------------------------- /tests/test_config.rs: -------------------------------------------------------------------------------- 1 | mod utils; 2 | use assert_cmd::prelude::*; 3 | use httpmock::MockServer; 4 | use predicates::prelude::*; 5 | use std::process::Command; 6 | use utils::{setup_tmp_directory, teardown_tmp_directory}; 7 | 8 | #[test] 9 | /// send a single valid request, expect a 200 response 10 | fn read_in_config_file_for_settings() -> Result<(), Box> { 11 | let srv = MockServer::start(); 12 | 13 | let (tmp_dir, file) = setup_tmp_directory(&["threads = 37".to_string()], "ferox-config.toml")?; 14 | 15 | Command::cargo_bin("feroxbuster") 16 | .unwrap() 17 | .current_dir(&tmp_dir) 18 | .arg("--url") 19 | .arg(srv.url("/")) 20 | .arg("--wordlist") 21 | .arg(file.as_os_str()) 22 | .arg("-vvvv") 23 | .assert() 24 | .success() 25 | .stderr(predicate::str::contains("│ 37")); 26 | 27 | teardown_tmp_directory(tmp_dir); 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /tests/test_parser.rs: -------------------------------------------------------------------------------- 1 | use assert_cmd::Command; 2 | use predicates::prelude::*; 3 | 4 | #[test] 5 | /// specify an incorrect param (-fc) with --help after it on the command line 6 | /// old behavior printed 7 | /// error: Found argument '-c' which wasn't expected, or isn't valid in this context 8 | /// 9 | /// USAGE: 10 | /// feroxbuster --add-slash --url ... 11 | /// 12 | /// For more information try --help 13 | /// 14 | /// the new behavior we expect to see is to print the long form help message, of which 15 | /// Ludicrous speed... go! is near the bottom of that output, so we can test for that 16 | fn parser_incorrect_param_with_tack_tack_help() { 17 | Command::cargo_bin("feroxbuster") 18 | .unwrap() 19 | .arg("-fc") 20 | .arg("--help") 21 | .assert() 22 | .success() 23 | .stdout(predicate::str::contains("Ludicrous speed... go!")); 24 | } 25 | 26 | #[test] 27 | /// specify an incorrect param (-fc) with --help after it on the command line 28 | /// old behavior printed 29 | /// error: Found argument '-c' which wasn't expected, or isn't valid in this context 30 | /// 31 | /// USAGE: 32 | /// feroxbuster --add-slash --url ... 33 | /// 34 | /// For more information try --help 35 | /// 36 | /// the new behavior we expect to see is to print the short form help message, of which 37 | /// "[CAUTION] 4 -v's is probably too much" is near the bottom of that output, so we can test for that 38 | fn parser_incorrect_param_with_tack_h() { 39 | Command::cargo_bin("feroxbuster") 40 | .unwrap() 41 | .arg("-fc") 42 | .arg("-h") 43 | .assert() 44 | .success() 45 | .stdout( 46 | predicate::str::contains("[CAUTION]") 47 | .and(predicate::str::contains("4")) 48 | .and(predicate::str::contains("-v's")) 49 | .and(predicate::str::contains("is")) 50 | .and(predicate::str::contains("probably")) 51 | .and(predicate::str::contains("too")) 52 | .and(predicate::str::contains("much")), 53 | ); 54 | } 55 | -------------------------------------------------------------------------------- /tests/test_scan_manager.rs: -------------------------------------------------------------------------------- 1 | mod utils; 2 | use assert_cmd::Command; 3 | use httpmock::Method::GET; 4 | use httpmock::MockServer; 5 | use predicates::prelude::*; 6 | use std::fs::{read_to_string, write}; 7 | use std::path::Path; 8 | use std::time; 9 | use utils::{setup_tmp_directory, teardown_tmp_directory}; 10 | 11 | #[test] 12 | /// pass a known serialized scan with 1 scan complete and 1 not. expect the incomplete scan to 13 | /// start and the complete to not start. expect the responses, scans, and configuration structures 14 | /// to be populated based off the contents of the given state file 15 | fn resume_scan_works() { 16 | let srv = MockServer::start(); 17 | let (tmp_dir, file) = 18 | setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap(); 19 | 20 | // localhost:PORT/ <- complete 21 | // localhost:PORT/js <- will get scanned with /css and /stuff 22 | let complete_scan = format!( 23 | r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","normalized_url":"{}","scan_type":"Directory","status":"Complete","num_requests":4174,"requests_made_so_far":0}}"#, 24 | srv.url("/"), 25 | srv.url("/"), 26 | ); 27 | let incomplete_scan = format!( 28 | r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","normalized_url":"{}/","scan_type":"Directory","status":"NotStarted","num_requests":4174,"requests_made_so_far":0}}"#, 29 | srv.url("/js"), 30 | srv.url("/js") 31 | ); 32 | let scans = format!(r#""scans":[{complete_scan},{incomplete_scan}]"#); 33 | 34 | let config = format!( 35 | r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#, 36 | file.to_string_lossy(), 37 | srv.url("/") 38 | ); 39 | 40 | // // localhost:PORT/js/css has already been seen, expect not to be scanned 41 | let response = format!( 42 | r#"{{"type":"response","url":"{}","path":"/js/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}"#, 43 | srv.url("/js/css") 44 | ); 45 | let responses = format!(r#""responses":[{response}]"#); 46 | 47 | // not scanned because /js is not complete, and /js/stuff response is not known 48 | let not_scanned_yet = srv.mock(|when, then| { 49 | when.method(GET).path("/js/stuff"); 50 | then.status(200).body("i expect to be scanned"); 51 | }); 52 | 53 | // will get scanned because /js is not complete, but because response of /js/css is known, the 54 | // response will not be in stdout 55 | let already_scanned = srv.mock(|when, then| { 56 | when.method(GET).path("/js/css"); 57 | then.status(200); 58 | }); 59 | 60 | // already scanned because scan on / is complete 61 | let also_already_scanned = srv.mock(|when, then| { 62 | when.method(GET).path("/css"); 63 | then.status(200).body("two words"); 64 | }); 65 | 66 | let state_file_contents = format!("{{{scans},{config},{responses}}}"); 67 | 68 | let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap(); 69 | 70 | Command::cargo_bin("feroxbuster") 71 | .unwrap() 72 | .arg("-vvv") 73 | .arg("--resume-from") 74 | .arg(state_file.as_os_str()) 75 | .assert() 76 | .success() 77 | .stdout( 78 | predicate::str::contains("/js/stuff") 79 | .and(predicate::str::contains("22c")) 80 | .and(predicate::str::contains("5w")) 81 | .and(predicate::str::contains("/js/css")) 82 | .not() 83 | .and(predicate::str::contains("2w")) 84 | .not() 85 | .and(predicate::str::contains("9c")) 86 | .not(), 87 | ); 88 | 89 | teardown_tmp_directory(tmp_dir); 90 | teardown_tmp_directory(tmp_dir2); 91 | 92 | assert_eq!(already_scanned.hits(), 1); 93 | assert_eq!(also_already_scanned.hits(), 0); 94 | assert_eq!(not_scanned_yet.hits(), 1); 95 | } 96 | 97 | #[test] 98 | /// kick off scan with a time limit; 99 | fn time_limit_enforced_when_specified() { 100 | let t1 = MockServer::start(); 101 | let t2 = MockServer::start(); 102 | 103 | let (tmp_dir, file) = 104 | setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap(); 105 | let (tgt_tmp_dir, targets) = 106 | setup_tmp_directory(&[t1.url("/"), t2.url("/")], "targets").unwrap(); 107 | 108 | // ensure the command will run long enough by adding crap to the wordlist 109 | let more_words = read_to_string(Path::new("tests/extra-words")).unwrap(); 110 | write(&file, more_words).unwrap(); 111 | 112 | assert!(file.metadata().unwrap().len() > 100); // sanity check on wordlist size 113 | 114 | let now = time::Instant::now(); 115 | let lower_bound = time::Duration::new(5, 0); 116 | let upper_bound = time::Duration::new(6, 0); 117 | 118 | Command::cargo_bin("feroxbuster") 119 | .unwrap() 120 | .arg("--stdin") 121 | .arg("--wordlist") 122 | .arg(file.as_os_str()) 123 | .arg("--time-limit") 124 | .arg("5s") 125 | .pipe_stdin(targets) 126 | .unwrap() 127 | .assert() 128 | .failure(); 129 | 130 | // expected run time is somewhere in the 30 seconds ballpark (real 0m37.376s) 131 | // so if the cmd returns in a significantly shorter amount of time, the test will have 132 | // succeeded 133 | 134 | // --time-limit is 5 seconds, so elapsed should be in a window that is greater than 5 135 | // but significantly less than 30ish 136 | assert!(now.elapsed() > lower_bound && now.elapsed() < upper_bound); 137 | 138 | teardown_tmp_directory(tmp_dir); 139 | teardown_tmp_directory(tgt_tmp_dir); 140 | } 141 | -------------------------------------------------------------------------------- /tests/utils/mod.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{remove_dir_all, write}; 2 | use std::path::PathBuf; 3 | use tempfile::TempDir; 4 | 5 | /// integration test helper: creates a temp directory, and writes `words` to 6 | /// a file named `filename` in the temp directory 7 | pub fn setup_tmp_directory( 8 | words: &[String], 9 | filename: &str, 10 | ) -> Result<(TempDir, PathBuf), Box> { 11 | let tmp_dir = TempDir::new()?; 12 | let file = tmp_dir.path().join(filename); 13 | write(&file, words.join("\n"))?; 14 | Ok((tmp_dir, file)) 15 | } 16 | 17 | /// integration test helper: removes a temporary directory, presumably created with 18 | /// [setup_tmp_directory](fn.setup_tmp_directory.html) 19 | pub fn teardown_tmp_directory(directory: TempDir) { 20 | remove_dir_all(directory).unwrap(); 21 | } 22 | --------------------------------------------------------------------------------