├── book ├── .gitignore ├── src │ ├── user-guide │ │ ├── commands │ │ │ ├── README.md │ │ │ └── brute.md │ │ ├── quickstart │ │ │ ├── README.md │ │ │ ├── usage │ │ │ │ ├── README.md │ │ │ │ ├── crate.md │ │ │ │ └── docker.md │ │ │ └── install.md │ │ ├── README.md │ │ └── environments.md │ ├── development │ │ ├── README.md │ │ ├── components │ │ │ ├── README.md │ │ │ └── extractors.md │ │ ├── generics │ │ │ ├── engine.md │ │ │ └── integration.md │ │ └── environment.md │ ├── SUMMARY.md │ └── README.md └── book.toml ├── .hadolint.yaml ├── tests ├── components │ ├── types │ │ ├── mod.rs │ │ └── result │ │ │ └── mod.rs │ ├── pools │ │ └── mod.rs │ ├── utilities │ │ └── mod.rs │ ├── extractors │ │ ├── mod.rs │ │ ├── regex_test.rs │ │ ├── html_test.rs │ │ └── json_test.rs │ ├── requesters │ │ └── mod.rs │ ├── cli │ │ ├── mod.rs │ │ ├── commands │ │ │ ├── module │ │ │ │ ├── mod.rs │ │ │ │ ├── list_test.rs │ │ │ │ └── get_test.rs │ │ │ └── mod.rs │ │ └── cli_test.rs │ ├── modules │ │ ├── engines │ │ │ ├── mod.rs │ │ │ ├── bing_test.rs │ │ │ ├── yahoo_test.rs │ │ │ ├── google_test.rs │ │ │ └── duckduckgo_test.rs │ │ ├── generics │ │ │ └── mod.rs │ │ ├── integrations │ │ │ ├── mod.rs │ │ │ ├── dnsrepo_test.rs │ │ │ ├── digitorus_test.rs │ │ │ ├── sitedossier_test.rs │ │ │ ├── hackertarget_test.rs │ │ │ ├── waybackarchive_test.rs │ │ │ ├── crtsh_test.rs │ │ │ ├── anubis_test.rs │ │ │ ├── leakix_test.rs │ │ │ ├── alienvault_test.rs │ │ │ ├── threatcrowd_test.rs │ │ │ ├── subdomaincenter_test.rs │ │ │ ├── chaos_test.rs │ │ │ ├── bevigil_test.rs │ │ │ ├── netcraft_test.rs │ │ │ ├── bufferover_test.rs │ │ │ ├── whoisxmlapi_test.rs │ │ │ ├── dnsdumpsterapi_test.rs │ │ │ ├── securitytrails_test.rs │ │ │ ├── censys_test.rs │ │ │ ├── certspotter_test.rs │ │ │ └── binaryedge_test.rs │ │ └── mod.rs │ ├── common │ │ ├── mod.rs │ │ ├── mock │ │ │ ├── mod.rs │ │ │ ├── resolver.rs │ │ │ └── modules.rs │ │ └── constants.rs │ ├── main.rs │ └── resolver_test.rs ├── stubs │ ├── hello │ │ ├── hello.json │ │ ├── hello-delayed.json │ │ ├── hello-with-basic-http-auth.json │ │ └── hello-with-headers.json │ └── module │ │ ├── integrations │ │ ├── hackertarget.json │ │ ├── sitedossier.json │ │ ├── anubis.json │ │ ├── dnsdumpstercrawler │ │ │ ├── dnsdumpstercrawler-no-token.json │ │ │ ├── dnsdumpstercrawler-delayed.json │ │ │ └── dnsdumpstercrawler-with-token.json │ │ ├── subdomaincenter.json │ │ ├── dnsrepo.json │ │ ├── netcraft.json │ │ ├── digitorus.json │ │ ├── threatcrowd.json │ │ ├── commoncrawl │ │ │ ├── commoncrawl-index-no-data.json │ │ │ ├── commoncrawl-index-delayed.json │ │ │ ├── commoncrawl-cdx-4.json │ │ │ ├── commoncrawl-cdx-1.json │ │ │ ├── commoncrawl-cdx-2.json │ │ │ ├── commoncrawl-cdx-3.json │ │ │ └── commoncrawl-index-template.json │ │ ├── alienvault.json │ │ ├── leakix.json │ │ ├── github │ │ │ ├── github-code-search-results.json │ │ │ ├── github-code-search-no-data.json │ │ │ ├── github-code-search-delayed.json │ │ │ └── github-code-search-template.json │ │ ├── netlas │ │ │ ├── netlas-no-count.json │ │ │ ├── netlas-delayed.json │ │ │ └── with-count │ │ │ │ ├── netlas-count.json │ │ │ │ └── netlas-domains-download.json │ │ ├── bevigil.json │ │ ├── securitytrails.json │ │ ├── chaos.json │ │ ├── dnsdumpsterapi.json │ │ ├── binaryedge.json │ │ ├── bufferover.json │ │ ├── zoomeye.json │ │ ├── virustotal.json │ │ ├── builtwith.json │ │ ├── whoisxmlapi.json │ │ ├── waybackarchive │ │ │ ├── waybackarchive.json │ │ │ └── waybackarchive-delayed.json │ │ ├── crtsh.json │ │ ├── certspotter.json │ │ ├── censys.json │ │ └── shodan.json │ │ ├── generics │ │ ├── integration-with-no-auth.json │ │ ├── integration-with-invalid-data.json │ │ ├── integration-with-no-auth-delayed.json │ │ ├── search-engine.json │ │ ├── integration-with-basic-http-auth.json │ │ ├── integration-with-header-auth.json │ │ ├── integration-with-query-auth.json │ │ └── search-engine-delayed.json │ │ └── engines │ │ ├── bing.json │ │ ├── google.json │ │ ├── duckduckgo.json │ │ └── yahoo.json └── cache_test.rs ├── .github ├── FUNDING.yml ├── workflows │ ├── release-plz.yml │ ├── tests.yml │ ├── mdbook.yml │ ├── linters.yml │ ├── security.yml │ └── coverage.yml ├── CONTRIBUTING.md └── dependabot.yml ├── testing └── testdata │ ├── txt │ ├── wordlist.txt │ └── resolverlist.txt │ ├── html │ ├── subdomains.html │ └── subdomains-with-removes.html │ └── json │ └── subdomains.json ├── .codespellrc ├── .mdlrc ├── commitlint.config.js ├── assets ├── logo-dark.png └── logo-light.png ├── _typos.toml ├── src ├── modules │ ├── generics │ │ └── mod.rs │ ├── mod.rs │ ├── engines │ │ ├── mod.rs │ │ ├── bing.rs │ │ ├── google.rs │ │ └── yahoo.rs │ └── integrations │ │ └── mod.rs ├── requesters │ └── mod.rs ├── pools │ └── mod.rs ├── extractors │ └── mod.rs ├── interfaces │ ├── mod.rs │ ├── lookup.rs │ ├── extractor.rs │ └── requester.rs ├── types │ ├── config │ │ ├── mod.rs │ │ └── pool.rs │ ├── result │ │ ├── output.rs │ │ ├── mod.rs │ │ ├── pool.rs │ │ ├── metadata.rs │ │ └── statistics.rs │ ├── mod.rs │ ├── core.rs │ ├── func.rs │ └── filters.rs ├── enums │ ├── mod.rs │ ├── cache.rs │ ├── output.rs │ ├── result.rs │ └── auth.rs ├── utilities │ ├── mod.rs │ ├── serializers.rs │ ├── env.rs │ ├── regex.rs │ ├── http.rs │ └── cli.rs ├── cli │ ├── commands │ │ ├── module │ │ │ ├── list.rs │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── run.rs │ │ ├── mod.rs │ │ └── brute.rs │ ├── banner.rs │ └── mod.rs ├── logger.rs ├── bin │ └── subscan.rs └── constants.rs ├── resolverlist.template ├── rustfmt.toml ├── .gitignore ├── codecov.yml ├── .release-plz.toml ├── .env.template ├── examples ├── crate_run_usage.rs ├── custom_extractor.rs ├── crate_brute_usage.rs ├── custom_requester.rs ├── crate_scan_usage.rs └── custom_module.rs ├── dist-workspace.toml ├── Dockerfile ├── .dockerignore ├── LICENSE ├── deny.toml └── Cargo.toml /book/.gitignore: -------------------------------------------------------------------------------- 1 | book 2 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | ignored: 2 | - DL3008 3 | -------------------------------------------------------------------------------- /tests/components/types/mod.rs: -------------------------------------------------------------------------------- 1 | mod result; 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | buy_me_a_coffee: eredotpkfr 2 | -------------------------------------------------------------------------------- /testing/testdata/txt/wordlist.txt: -------------------------------------------------------------------------------- 1 | foo 2 | bar 3 | baz 4 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | ignore-words-list = crate,ser,statics 3 | -------------------------------------------------------------------------------- /.mdlrc: -------------------------------------------------------------------------------- 1 | rules "~MD033", "~MD013", "~MD002", "~MD007", "~MD034", "~MD029" 2 | -------------------------------------------------------------------------------- /tests/components/pools/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/pools"); 2 | -------------------------------------------------------------------------------- /tests/components/utilities/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/utilities"); 2 | -------------------------------------------------------------------------------- /tests/components/extractors/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/extractors"); 2 | -------------------------------------------------------------------------------- /tests/components/requesters/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/requesters"); 2 | -------------------------------------------------------------------------------- /tests/components/types/result/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/types/result"); 2 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | export default { extends: ['@commitlint/config-conventional'] }; 2 | -------------------------------------------------------------------------------- /tests/components/cli/mod.rs: -------------------------------------------------------------------------------- 1 | mod commands; 2 | 3 | automod::dir!("tests/components/cli"); 4 | -------------------------------------------------------------------------------- /assets/logo-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eredotpkfr/subscan/HEAD/assets/logo-dark.png -------------------------------------------------------------------------------- /assets/logo-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eredotpkfr/subscan/HEAD/assets/logo-light.png -------------------------------------------------------------------------------- /tests/components/modules/engines/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/modules/engines"); 2 | -------------------------------------------------------------------------------- /tests/components/modules/generics/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/modules/generics"); 2 | -------------------------------------------------------------------------------- /tests/components/cli/commands/module/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/cli/commands/module"); 2 | -------------------------------------------------------------------------------- /_typos.toml: -------------------------------------------------------------------------------- 1 | [files] 2 | extend-exclude = ["Cargo.lock", "cliff.toml", ".codespellrc", "CHANGELOG.md"] 3 | -------------------------------------------------------------------------------- /tests/components/common/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod constants; 2 | pub mod mock; 3 | pub mod stub; 4 | pub mod utils; 5 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/mod.rs: -------------------------------------------------------------------------------- 1 | automod::dir!("tests/components/modules/integrations"); 2 | -------------------------------------------------------------------------------- /tests/components/cli/commands/mod.rs: -------------------------------------------------------------------------------- 1 | mod module; 2 | 3 | automod::dir!("tests/components/cli/commands"); 4 | -------------------------------------------------------------------------------- /tests/components/common/mock/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod dns; 2 | pub mod funcs; 3 | pub mod modules; 4 | pub mod resolver; 5 | -------------------------------------------------------------------------------- /tests/components/modules/mod.rs: -------------------------------------------------------------------------------- 1 | mod engines; 2 | mod generics; 3 | mod integrations; 4 | 5 | automod::dir!("tests/components/modules"); 6 | -------------------------------------------------------------------------------- /src/modules/generics/mod.rs: -------------------------------------------------------------------------------- 1 | /// Generic search engine module 2 | pub mod engine; 3 | /// Generic API integration module 4 | pub mod integration; 5 | -------------------------------------------------------------------------------- /src/requesters/mod.rs: -------------------------------------------------------------------------------- 1 | /// Chrome browser to send HTTP requests via Chrome 2 | pub mod chrome; 3 | /// Basic HTTP client, uses [`reqwest`] 4 | pub mod client; 5 | -------------------------------------------------------------------------------- /resolverlist.template: -------------------------------------------------------------------------------- 1 | 127.0.0.1:53 2 | 192.168.1.43:8080 3 | 10.126.125.98:4444 4 | [::1]:53 5 | [2001:db8::1]:8080 6 | [2001:db8:85a3:8d3:1319:8a2e:370:7348]:4444 7 | -------------------------------------------------------------------------------- /src/pools/mod.rs: -------------------------------------------------------------------------------- 1 | /// [`SubscanBrutePool`](crate::pools::brute::SubscanBrutePool) utilities 2 | pub mod brute; 3 | /// [`SubscanModulePool`](crate::pools::module::SubscanModulePool) utilities 4 | pub mod module; 5 | -------------------------------------------------------------------------------- /tests/components/main.rs: -------------------------------------------------------------------------------- 1 | mod cli; 2 | mod common; 3 | mod extractors; 4 | mod modules; 5 | mod pools; 6 | mod requesters; 7 | mod types; 8 | mod utilities; 9 | 10 | automod::dir!("tests/components"); 11 | -------------------------------------------------------------------------------- /src/extractors/mod.rs: -------------------------------------------------------------------------------- 1 | /// Subdomain extractor for HTML documents 2 | pub mod html; 3 | /// JSON extractor to extract subdomains from JSON content 4 | pub mod json; 5 | /// Extract subdomains with regex statement 6 | pub mod regex; 7 | -------------------------------------------------------------------------------- /src/modules/mod.rs: -------------------------------------------------------------------------------- 1 | /// Search engine modules 2 | pub mod engines; 3 | /// Generic module implementations 4 | pub mod generics; 5 | /// Integration modules 6 | pub mod integrations; 7 | /// Check zone transfer 8 | pub mod zonetransfer; 9 | -------------------------------------------------------------------------------- /src/interfaces/mod.rs: -------------------------------------------------------------------------------- 1 | /// Extractor trait definition 2 | pub mod extractor; 3 | /// IP lookup future 4 | pub mod lookup; 5 | /// `Subscan` module trait definition 6 | pub mod module; 7 | /// HTTP requester trait definition 8 | pub mod requester; 9 | -------------------------------------------------------------------------------- /src/types/config/mod.rs: -------------------------------------------------------------------------------- 1 | /// Subscan thread pool configurations 2 | pub mod pool; 3 | /// Requester configurations 4 | pub mod requester; 5 | /// DNS resolver configurations 6 | pub mod resolver; 7 | /// Subscan configurations 8 | pub mod subscan; 9 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Configuration for https://rust-lang.github.io/rustfmt/ 2 | 3 | edition = "2024" 4 | max_width = 100 5 | chain_width = 80 6 | use_field_init_shorthand = true 7 | use_try_shorthand = true 8 | unstable_features = true 9 | group_imports = "StdExternalCrate" 10 | imports_granularity = "Crate" 11 | -------------------------------------------------------------------------------- /tests/stubs/hello/hello.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/hello" 5 | }, 6 | "response": { 7 | "body": "hello", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/enums/mod.rs: -------------------------------------------------------------------------------- 1 | /// Authentication variants 2 | pub mod auth; 3 | /// Cache related enumerations 4 | pub mod cache; 5 | /// Content types 6 | pub mod content; 7 | /// Dispatcher definitions 8 | pub mod dispatchers; 9 | /// Output format options 10 | pub mod output; 11 | /// Result data types 12 | pub mod result; 13 | -------------------------------------------------------------------------------- /testing/testdata/html/subdomains.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
6 |
7 | 8 | bar.foo.com 9 | baz.foo.com 10 | 11 |
12 |
13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/modules/engines/mod.rs: -------------------------------------------------------------------------------- 1 | /// Enumerate subdomains on `Bing` by using dorking 2 | pub mod bing; 3 | /// Enumerate subdomains on `DuckDuckGo` by using dorking 4 | pub mod duckduckgo; 5 | /// Enumerate subdomains on `Google` by using dorking 6 | pub mod google; 7 | /// Enumerate subdomains on `Yahoo` by using dorking 8 | pub mod yahoo; 9 | -------------------------------------------------------------------------------- /src/types/result/output.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | 3 | use derive_more::From; 4 | 5 | /// Output file type, it stores file name and object 6 | #[derive(From)] 7 | pub struct OutputFile { 8 | /// File name 9 | pub name: String, 10 | /// File object to write or read something 11 | pub descriptor: File, 12 | } 13 | -------------------------------------------------------------------------------- /src/utilities/mod.rs: -------------------------------------------------------------------------------- 1 | /// Helpful functions related with CLI 2 | pub mod cli; 3 | /// Environment related utilities 4 | pub mod env; 5 | /// HTTP related utilities 6 | pub mod http; 7 | /// Network utilities 8 | pub mod net; 9 | /// Regex utilities 10 | pub mod regex; 11 | /// JSON serializer utilities 12 | pub mod serializers; 13 | -------------------------------------------------------------------------------- /testing/testdata/html/subdomains-with-removes.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
6 |
7 | 8 | bar
.foo.com 9 | baz.foo.
com 10 | 11 |
12 |
13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/hackertarget.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/hackertarget" 5 | }, 6 | "response": { 7 | "body": "bar.foo.com\n", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # These are backup files generated by rustfmt 7 | **/*.rs.bk 8 | 9 | # MSVC Windows builds of rustc generate these, which store debugging information 10 | *.pdb 11 | 12 | # Added by cargo 13 | /target 14 | 15 | # Ignore local .env file 16 | .env 17 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/sitedossier.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/sitedossier" 5 | }, 6 | "response": { 7 | "body": "
  1. bar.foo.com
", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/types/result/mod.rs: -------------------------------------------------------------------------------- 1 | /// Result item definitions 2 | pub mod item; 3 | /// Subscan result metadata type 4 | pub mod metadata; 5 | /// Output file types 6 | pub mod output; 7 | /// Pool result definitions 8 | pub mod pool; 9 | /// Statistics types 10 | pub mod statistics; 11 | /// Status types 12 | pub mod status; 13 | /// Scan result types 14 | pub mod subscan; 15 | -------------------------------------------------------------------------------- /tests/stubs/hello/hello-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/hello-delayed" 5 | }, 6 | "response": { 7 | "body": "hello", 8 | "fixedDelayMilliseconds": 1000, 9 | "headers": { 10 | "content-type": "text/html" 11 | }, 12 | "status": 200 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/anubis.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/anubis" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": [ 11 | "bar.foo.com" 12 | ], 13 | "status": 200 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/dnsdumpstercrawler/dnsdumpstercrawler-no-token.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/dnsdumpstercrawler-no-token" 5 | }, 6 | "response": { 7 | "body": "foo", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/subdomaincenter.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/subdomaincenter" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": [ 11 | "bar.foo.com" 12 | ], 13 | "status": 200 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/dnsrepo.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/dnsrepo" 5 | }, 6 | "response": { 7 | "body": "
bar.foo.com
", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/netcraft.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/netcraft" 5 | }, 6 | "response": { 7 | "body": "
bar.foo.com
", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/digitorus.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/digitorus" 5 | }, 6 | "response": { 7 | "body": "
bar.foo.com
", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/types/mod.rs: -------------------------------------------------------------------------------- 1 | /// Type definitions are related with any configuration 2 | pub mod config; 3 | /// Core type definitions 4 | pub mod core; 5 | /// Types that related environments 6 | pub mod env; 7 | /// Filter definitions 8 | pub mod filters; 9 | /// Function type definitions 10 | pub mod func; 11 | /// Query types for search engines 12 | pub mod query; 13 | /// `Subscan` module result types 14 | pub mod result; 15 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/dnsdumpstercrawler/dnsdumpstercrawler-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/dnsdumpstercrawler-delayed" 5 | }, 6 | "response": { 7 | "body": "foo", 8 | "fixedDelayMilliseconds": 1000, 9 | "headers": { 10 | "content-type": "text/html" 11 | }, 12 | "status": 200 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/components/common/constants.rs: -------------------------------------------------------------------------------- 1 | pub const LOCAL_HOST: &str = "127.0.0.1"; 2 | pub const TEST_URL: &str = "http://foo.com"; 3 | pub const TEST_DOMAIN: &str = "foo.com"; 4 | pub const TEST_FOO_SUBDOMAIN: &str = "foo.foo.com"; 5 | pub const TEST_BAR_SUBDOMAIN: &str = "bar.foo.com"; 6 | pub const TEST_BAZ_SUBDOMAIN: &str = "baz.foo.com"; 7 | pub const TEST_API_KEY: &str = "test-api-key"; 8 | pub const READ_ERROR: &str = "Cannot read file!"; 9 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/threatcrowd.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/threatcrowd" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": { 11 | "subdomains": [ 12 | "bar.foo.com" 13 | ] 14 | }, 15 | "status": 200 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/integration-with-no-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/subdomains" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": { 11 | "subdomains": [ 12 | "bar.foo.com" 13 | ] 14 | }, 15 | "status": 200 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/stubs/hello/hello-with-basic-http-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "basicAuth": { 4 | "password": "bar", 5 | "username": "foo" 6 | }, 7 | "method": "GET", 8 | "urlPath": "/hello-with-basic-http-auth" 9 | }, 10 | "response": { 11 | "body": "hello", 12 | "headers": { 13 | "content-type": "text/html" 14 | }, 15 | "status": 200 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-index-no-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/commoncrawl/index-no-data" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": {}, 11 | "status": 200, 12 | "transformers": [ 13 | "response-template" 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/dnsdumpstercrawler/dnsdumpstercrawler-with-token.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "ANY", 4 | "urlPath": "/dnsdumpstercrawler-with-token" 5 | }, 6 | "response": { 7 | "body": "\"Authorization\": \"foo\"
bar.foo.com
", 8 | "headers": { 9 | "content-type": "text/html" 10 | }, 11 | "status": 200 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/integration-with-invalid-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/subdomains" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": { 11 | "subdomains": [ 12 | "bar.foo.com", 13 | "foo.bar.com" 14 | ] 15 | }, 16 | "status": 200 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/integration-with-no-auth-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/subdomains" 5 | }, 6 | "response": { 7 | "fixedDelayMilliseconds": 1000, 8 | "headers": { 9 | "content-type": "application/json" 10 | }, 11 | "jsonBody": { 12 | "subdomains": [ 13 | "bar.foo.com" 14 | ] 15 | }, 16 | "status": 200 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/alienvault.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/alienvault" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": { 11 | "passive_dns": [ 12 | { 13 | "hostname": "bar.foo.com" 14 | } 15 | ] 16 | }, 17 | "status": 200 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /testing/testdata/json/subdomains.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "subdomains": [ 4 | { 5 | "ip": "127.0.0.1", 6 | "subdomain": "bar.foo.com" 7 | }, 8 | { 9 | "ip": "127.0.0.1", 10 | "subdomain": "baz.foo.com" 11 | } 12 | ] 13 | }, 14 | "id": "7a3db2d9-1713-4e72-8261-5976fa5b9cf9", 15 | "scan_time": "2024-09-22T11:15:00.430218", 16 | "status": "SUCCESS" 17 | } 18 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-index-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/commoncrawl/index-delayed" 5 | }, 6 | "response": { 7 | "fixedDelayMilliseconds": 1000, 8 | "headers": { 9 | "content-type": "application/json" 10 | }, 11 | "jsonBody": {}, 12 | "status": 200, 13 | "transformers": [ 14 | "response-template" 15 | ] 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /book/src/user-guide/commands/README.md: -------------------------------------------------------------------------------- 1 | # Commands 2 | 3 | This chapter provides a comprehensive guide to the `Subscan` CLI commands. Below is a list of available commands. For detailed information on usage and arguments, refer to the corresponding sections 4 | 5 | 6 | 7 | - [scan](scan.md) 8 | - [brute](brute.md) 9 | - [module](module.md) 10 | - [list](module.md#list) 11 | - [get](module.md#get) 12 | - [run](module.md#run) 13 | 14 | 15 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/leakix.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/leakix" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": [ 11 | { 12 | "distinct_ips": 1, 13 | "last_seen": "2021-05-13T10:28:34.356Z", 14 | "subdomain": "bar.foo.com" 15 | } 16 | ], 17 | "status": 200 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /book/src/user-guide/quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | `Subscan` is a fast and efficient subdomain enumeration tool designed for penetration testers and security researchers. In this chapter, you'll learn how to quickly set up and start using `Subscan` to discover subdomains and improve your security assessments 4 | 5 | Here's what you'll find in this chapter 6 | 7 | - [Installing Subscan](install.md) 8 | - [Usage Methods and Scenarios](usage/index.html) 9 | - [Details of Commands and Typical Use Cases](../commands/index.html) 10 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-cdx-4.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "url": { 6 | "equalTo": "*.foo.com" 7 | } 8 | }, 9 | "urlPath": "/commoncrawl/cdx-4" 10 | }, 11 | "response": { 12 | "body": "bar.foo.com", 13 | "fixedDelayMilliseconds": 100, 14 | "headers": { 15 | "content-type": "text/html" 16 | }, 17 | "status": 200 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/github/github-code-search-results.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "token github-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/github-code-search/results" 10 | }, 11 | "response": { 12 | "body": "this is bar.foo.com content", 13 | "headers": { 14 | "content-type": "text/html" 15 | }, 16 | "status": 200 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/netlas/netlas-no-count.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-API-Key": { 5 | "equalTo": "netlas-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/netlas-no-count" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "no-count": "foo" 17 | }, 18 | "status": 200 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/search-engine.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "num": { 6 | "equalTo": 100 7 | }, 8 | "q": { 9 | "equalTo": "site:foo.com" 10 | } 11 | }, 12 | "urlPath": "/search" 13 | }, 14 | "response": { 15 | "body": "bar.foo.com", 16 | "headers": { 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/engines/bing.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "num": { 6 | "equalTo": 100 7 | }, 8 | "q": { 9 | "equalTo": "site:foo.com" 10 | } 11 | }, 12 | "urlPath": "/search" 13 | }, 14 | "response": { 15 | "body": "
bar.foo.com
", 16 | "headers": { 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/engines/google.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "num": { 6 | "equalTo": 100 7 | }, 8 | "q": { 9 | "equalTo": "site:foo.com" 10 | } 11 | }, 12 | "urlPath": "/search" 13 | }, 14 | "response": { 15 | "body": "
bar.foo.com
", 16 | "headers": { 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/integration-with-basic-http-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "basicAuth": { 4 | "password": "bar", 5 | "username": "foo" 6 | }, 7 | "method": "GET", 8 | "urlPath": "/subdomains" 9 | }, 10 | "response": { 11 | "headers": { 12 | "content-type": "application/json" 13 | }, 14 | "jsonBody": { 15 | "subdomains": [ 16 | "bar.foo.com" 17 | ] 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: # https://docs.codecov.com/docs/codecovyml-reference#coverage 2 | precision: 2 # e.g. 91.67% 3 | round: nearest 4 | range: 65..85 # https://docs.codecov.com/docs/coverage-configuration#section-range 5 | 6 | status: # https://docs.codecov.com/docs/commit-status 7 | project: 8 | default: 9 | target: auto 10 | threshold: "1%" 11 | if_ci_failed: error 12 | branches: 13 | - main 14 | patch: off 15 | 16 | ignore: 17 | - "src/lib.rs" 18 | - "src/logger.rs" 19 | - "src/bin/subscan.rs" 20 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/bevigil.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-Access-Token": { 5 | "equalTo": "bevigil-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/bevigil" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "subdomains": [ 17 | "bar.foo.com" 18 | ] 19 | }, 20 | "status": 200 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/components/cli/cli_test.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use subscan::cli::Cli; 3 | 4 | #[tokio::test] 5 | #[should_panic] 6 | async fn cli_parse_error_test() { 7 | Cli::try_parse_from(vec!["subscan", "-x"]).unwrap(); 8 | } 9 | 10 | #[tokio::test] 11 | async fn verbosity_test() { 12 | let args = vec!["subscan", "scan", "-d", "foo.com", "-qqqq"]; 13 | let cli = Cli::try_parse_from(args).unwrap(); 14 | 15 | cli.init().await; 16 | cli.banner().await; 17 | 18 | assert!(cli.verbose.is_present()); 19 | assert_eq!(cli.verbose.to_string(), "off"); 20 | } 21 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/integration-with-header-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-API-Key": { 5 | "equalTo": "test-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/subdomains" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "subdomains": [ 17 | "bar.foo.com" 18 | ] 19 | }, 20 | "status": 200 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/netlas/netlas-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-API-Key": { 5 | "equalTo": "netlas-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/api/domains_count/" 10 | }, 11 | "response": { 12 | "fixedDelayMilliseconds": 1000, 13 | "headers": { 14 | "content-type": "application/json" 15 | }, 16 | "jsonBody": { 17 | "no-count": "foo" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/securitytrails.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "APIKEY": { 5 | "equalTo": "securitytrails-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/securitytrails" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "status": 200, 17 | "subdomains": [ 18 | "bar" 19 | ] 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/integration-with-query-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "apikey": { 6 | "equalTo": "test-api-key" 7 | } 8 | }, 9 | "urlPath": "/subdomains" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "subdomains": [ 17 | "bar.foo.com" 18 | ] 19 | }, 20 | "status": 200 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/stubs/module/engines/duckduckgo.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "num": { 6 | "equalTo": 100 7 | }, 8 | "q": { 9 | "equalTo": "site:foo.com" 10 | } 11 | } 12 | }, 13 | "response": { 14 | "body": "
bar.foo.com
", 15 | "headers": { 16 | "content-type": "text/html" 17 | }, 18 | "status": 200 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/stubs/module/generics/search-engine-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "num": { 6 | "equalTo": 100 7 | }, 8 | "q": { 9 | "equalTo": "site:foo.com" 10 | } 11 | }, 12 | "urlPath": "/search" 13 | }, 14 | "response": { 15 | "body": "bar.foo.com", 16 | "fixedDelayMilliseconds": 1000, 17 | "headers": { 18 | "content-type": "text/html" 19 | }, 20 | "status": 200 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/chaos.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "chaos-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/chaos" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "domain": "foo.com", 17 | "subdomains": [ 18 | "bar" 19 | ] 20 | }, 21 | "status": 200 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tests/stubs/hello/hello-with-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "x-api-key": { 5 | "equalTo": "hello-api" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "search": { 11 | "equalTo": "site:foo.com" 12 | } 13 | }, 14 | "urlPath": "/hello-with-headers" 15 | }, 16 | "response": { 17 | "body": "hello", 18 | "headers": { 19 | "content-type": "text/html" 20 | }, 21 | "status": 200 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tests/stubs/module/engines/yahoo.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "num": { 6 | "equalTo": 100 7 | }, 8 | "p": { 9 | "equalTo": "site:foo.com" 10 | } 11 | }, 12 | "urlPath": "/search" 13 | }, 14 | "response": { 15 | "body": "

  1. bar.foo.com

", 16 | "headers": { 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-cdx-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "url": { 6 | "equalTo": "*.foo.com" 7 | } 8 | }, 9 | "urlPath": "/commoncrawl/cdx-1" 10 | }, 11 | "response": { 12 | "body": "bar.foo.com", 13 | "headers": { 14 | "accept-ranges": "bytes", 15 | "connection": "keep-alive", 16 | "content-length": 11, 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-cdx-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "url": { 6 | "equalTo": "*.foo.com" 7 | } 8 | }, 9 | "urlPath": "/commoncrawl/cdx-2" 10 | }, 11 | "response": { 12 | "body": "baz.foo.com", 13 | "headers": { 14 | "accept-ranges": "bytes", 15 | "connection": "keep-alive", 16 | "content-length": 11, 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-cdx-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "url": { 6 | "equalTo": "*.foo.com" 7 | } 8 | }, 9 | "urlPath": "/commoncrawl/cdx-3" 10 | }, 11 | "response": { 12 | "body": "barbaz.foo.com", 13 | "headers": { 14 | "accept-ranges": "bytes", 15 | "connection": "keep-alive", 16 | "content-length": 11, 17 | "content-type": "text/html" 18 | }, 19 | "status": 200 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/dnsdumpsterapi.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-API-Key": { 5 | "equalTo": "dnsdumpsterapi-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/dnsdumpsterapi" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "a": [ 17 | { 18 | "host": "bar.foo.com" 19 | } 20 | ] 21 | }, 22 | "status": 200 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/cli/commands/module/list.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | 3 | use clap::Args; 4 | 5 | use crate::{types::core::SubscanModule, utilities::cli}; 6 | 7 | /// List command to list implemented modules 8 | #[derive(Args, Clone, Debug)] 9 | pub struct ModuleListSubCommandArgs {} 10 | 11 | impl ModuleListSubCommandArgs { 12 | pub async fn as_table(&self, modules: &Vec, out: &mut W) { 13 | let mut table = cli::create_module_table().await; 14 | 15 | for module in modules { 16 | table.add_row(module.lock().await.as_table_row().await); 17 | } 18 | 19 | table.print(out).unwrap(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /book/book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | authors = ["Erdoğan Yoksul "] 3 | language = "en" 4 | src = "src" 5 | title = "subscan" 6 | description = "A subdomain enumeration tool leveraging diverse techniques, designed for advanced pentesting operations" 7 | 8 | [rust] 9 | edition = "2021" 10 | 11 | [build] 12 | create-missing = false 13 | 14 | [output.html.search] 15 | limit-results = 15 16 | 17 | [output.html] 18 | default-theme = "ayu" 19 | git-repository-url = "https://github.com/eredotpkfr/subscan" 20 | cname = "www.erdoganyoksul.com" 21 | 22 | [output.html.playground] 23 | runnable = false 24 | 25 | [preprocessor.index] 26 | 27 | [preprocessor.links] 28 | -------------------------------------------------------------------------------- /tests/components/modules/engines/bing_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{modules::engines::bing::Bing, types::result::status::SubscanModuleStatus}; 2 | 3 | use crate::common::{ 4 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN}, 5 | mock::funcs, 6 | utils, 7 | }; 8 | 9 | #[tokio::test] 10 | #[stubr::mock("module/engines/bing.json")] 11 | async fn run_test() { 12 | let mut bing = Bing::dispatcher(); 13 | 14 | funcs::wrap_module_url(&mut bing, &stubr.path("/search")); 15 | 16 | let (results, status) = utils::run_module(bing, TEST_DOMAIN).await; 17 | 18 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 19 | assert_eq!(status, SubscanModuleStatus::Finished); 20 | } 21 | -------------------------------------------------------------------------------- /tests/components/modules/engines/yahoo_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{modules::engines::yahoo::Yahoo, types::result::status::SubscanModuleStatus}; 2 | 3 | use crate::common::{ 4 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN}, 5 | mock::funcs, 6 | utils, 7 | }; 8 | 9 | #[tokio::test] 10 | #[stubr::mock("module/engines/yahoo.json")] 11 | async fn run_test() { 12 | let mut yahoo = Yahoo::dispatcher(); 13 | 14 | funcs::wrap_module_url(&mut yahoo, &stubr.path("/search")); 15 | 16 | let (results, status) = utils::run_module(yahoo, TEST_DOMAIN).await; 17 | 18 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 19 | assert_eq!(status, SubscanModuleStatus::Finished); 20 | } 21 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/binaryedge.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-Key": { 5 | "equalTo": "binaryedge-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "page": { 11 | "absent": true 12 | } 13 | }, 14 | "urlPath": "/binaryedge" 15 | }, 16 | "response": { 17 | "headers": { 18 | "content-type": "application/json" 19 | }, 20 | "jsonBody": { 21 | "events": [ 22 | "bar.foo.com" 23 | ] 24 | }, 25 | "status": 200 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/netlas/with-count/netlas-count.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-API-Key": { 5 | "equalTo": "netlas-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "q": { 11 | "equalTo": "domain:*.foo.com AND NOT domain:foo.com" 12 | } 13 | }, 14 | "urlPath": "/api/domains_count/" 15 | }, 16 | "response": { 17 | "headers": { 18 | "content-type": "application/json" 19 | }, 20 | "jsonBody": { 21 | "count": 1 22 | }, 23 | "status": 200 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/components/modules/engines/google_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{modules::engines::google::Google, types::result::status::SubscanModuleStatus}; 2 | 3 | use crate::common::{ 4 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN}, 5 | mock::funcs, 6 | utils, 7 | }; 8 | 9 | #[tokio::test] 10 | #[stubr::mock("module/engines/google.json")] 11 | async fn run_test() { 12 | let mut google = Google::dispatcher(); 13 | 14 | funcs::wrap_module_url(&mut google, &stubr.path("/search")); 15 | 16 | let (results, status) = utils::run_module(google, TEST_DOMAIN).await; 17 | 18 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 19 | assert_eq!(status, SubscanModuleStatus::Finished); 20 | } 21 | -------------------------------------------------------------------------------- /.release-plz.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | # path of the git-cliff configuration 3 | changelog_config = "cliff.toml" 4 | 5 | # enable changelog updates 6 | changelog_update = true 7 | 8 | # update dependencies with `cargo update` 9 | dependencies_update = true 10 | 11 | # create tags for the releases 12 | git_tag_enable = true 13 | 14 | # disable GitHub releases 15 | git_release_enable = false 16 | 17 | # labels for the release PR 18 | pr_labels = ["release"] 19 | 20 | # disallow updating repositories with uncommitted changes 21 | allow_dirty = false 22 | 23 | # disallow packaging with uncommitted changes 24 | publish_allow_dirty = false 25 | 26 | # disable running `cargo-semver-checks` 27 | semver_check = false 28 | -------------------------------------------------------------------------------- /src/cli/commands/module/get.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | 3 | use clap::Args; 4 | use tokio::sync::Mutex; 5 | 6 | use crate::{enums::dispatchers::SubscanModuleDispatcher, utilities::cli}; 7 | 8 | /// Get command to fetch any module 9 | #[derive(Args, Clone, Debug)] 10 | pub struct ModuleGetSubCommandArgs { 11 | /// Module name to be fetched 12 | pub name: String, 13 | } 14 | 15 | impl ModuleGetSubCommandArgs { 16 | pub async fn as_table(&self, module: &Mutex, out: &mut W) { 17 | let mut table = cli::create_module_table().await; 18 | 19 | table.add_row(module.lock().await.as_table_row().await); 20 | table.print(out).unwrap(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/bufferover.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-API-Key": { 5 | "equalTo": "bufferover-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/bufferover" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "Meta": { 17 | "domain": "foo.com" 18 | }, 19 | "Results": [ 20 | "54.201.204.183,581faa6ff692d0ba8185753570c8624a2a6b4e8e47bd5322216cc12a41def044,,bar.foo.com" 21 | ] 22 | }, 23 | "status": 200 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/zoomeye.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "API-Key": { 5 | "equalTo": "zoomeye-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "page": { 11 | "absent": true 12 | } 13 | }, 14 | "urlPath": "/zoomeye" 15 | }, 16 | "response": { 17 | "headers": { 18 | "content-type": "application/json" 19 | }, 20 | "jsonBody": { 21 | "list": [ 22 | { 23 | "name": "bar.foo.com" 24 | } 25 | ] 26 | }, 27 | "status": 200 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/virustotal.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "X-APIKey": { 5 | "equalTo": "virustotal-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "cursor": { 11 | "absent": true 12 | } 13 | }, 14 | "urlPath": "/virustotal" 15 | }, 16 | "response": { 17 | "headers": { 18 | "content-type": "application/json" 19 | }, 20 | "jsonBody": { 21 | "data": [ 22 | { 23 | "id": "bar.foo.com" 24 | } 25 | ] 26 | }, 27 | "status": 200 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | # Bevigil 2 | SUBSCAN_BEVIGIL_APIKEY=foo 3 | # BinaryEdge 4 | SUBSCAN_BINARYEDGE_APIKEY=bar 5 | # BufferOver 6 | SUBSCAN_BUFFEROVER_APIKEY=baz 7 | # BuiltWith 8 | SUBSCAN_BUILTWITH_APIKEY=foo 9 | # Censys 10 | SUBSCAN_CENSYS_APIKEY=bar 11 | # CertSpotter 12 | SUBSCAN_CERTSPOTTER_APIKEY=baz 13 | # Chaos 14 | SUBSCAN_CHAOS_APIKEY=foo 15 | # Shodan 16 | SUBSCAN_SHODAN_APIKEY=bar 17 | # VirusTotal 18 | SUBSCAN_VIRUSTOTAL_APIKEY=baz 19 | # WhoisXMLAPI 20 | SUBSCAN_WHOISXMLAPI_APIKEY=foo 21 | # ZoomEye 22 | SUBSCAN_ZOOMEYE_APIKEY=bar 23 | # GitHub Access Token 24 | SUBSCAN_GITHUB_APIKEY=baz 25 | # Netlas 26 | SUBSCAN_NETLAS_APIKEY=foo 27 | # SecurityTrails 28 | SUBSCAN_SECURITYTRAILS_APIKEY=bar 29 | # DNSDumpsterAPI 30 | SUBSCAN_DNSDUMPSTERAPI_APIKEY=baz 31 | -------------------------------------------------------------------------------- /book/src/user-guide/quickstart/usage/README.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | This chapter guides you on how to use the `Subscan` to efficiently discover subdomains. Subdomain discovery features are implemented as modular `SubscanModule` components, which are automatically executed when a scan is initiated. For technical insights, check out the [Development](../../../development/index.html) chapter 4 | 5 | ✨ In this section, you'll find detailed instructions for different usage methods 6 | 7 | - How to use the [Subscan CLI](cli.md) for quick and effective subdomain enumeration 8 | - [Run Subscan in a Docker container](docker.md) for a lightweight, portable setup 9 | - Integrating [Subscan as a Crate](crate.md) in your Rust project for seamless integration with your codebase 10 | -------------------------------------------------------------------------------- /.github/workflows/release-plz.yml: -------------------------------------------------------------------------------- 1 | name: CI-CD 2 | 3 | permissions: 4 | pull-requests: write 5 | contents: write 6 | 7 | on: 8 | push: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | release-plz: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v6 18 | with: 19 | token: ${{ secrets.RELEASE_PLZ_TOKEN }} 20 | fetch-depth: 0 21 | 22 | - name: Install Rust toolchain 23 | uses: dtolnay/rust-toolchain@stable 24 | 25 | - name: Run release-plz 26 | uses: release-plz/action@v0.5 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.RELEASE_PLZ_TOKEN }} 29 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 30 | -------------------------------------------------------------------------------- /src/cli/commands/mod.rs: -------------------------------------------------------------------------------- 1 | /// Brute command to start brute force attack on any domain address 2 | pub mod brute; 3 | /// Module command to manage implemented modules 4 | pub mod module; 5 | /// Scan command to start scan on any domain address 6 | pub mod scan; 7 | 8 | use clap::Subcommand; 9 | 10 | use crate::cli::commands::{ 11 | brute::BruteCommandArgs, module::ModuleCommandArgs, scan::ScanCommandArgs, 12 | }; 13 | 14 | /// List of CLI commands 15 | #[derive(Clone, Debug, Subcommand)] 16 | pub enum Commands { 17 | /// Start scan on any domain address 18 | Scan(ScanCommandArgs), 19 | /// Start brute force attack with a given wordlist 20 | Brute(BruteCommandArgs), 21 | /// Subcommand to manage implemented modules 22 | Module(ModuleCommandArgs), 23 | } 24 | -------------------------------------------------------------------------------- /book/src/development/README.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | This chapter provides an in-depth guide for developers on how to contribute to and extend `Subscan`. It covers everything from setting up the development environment to understanding the core architecture and adding new features or modules. 4 | 5 | Here’s a quick overview of the sections included 6 | 7 | - [Setup Development Environment](environment.md) 8 | - [Components](components/index.html) 9 | - [Requesters](components/requesters.md) 10 | - [Extractors](components/extractors.md) 11 | - [Subscan Module](components/module.md) 12 | - [Generic Modules](generics/index.html) 13 | - [Integration](generics/integration.md) 14 | - [Search Engine](generics/engine.md) 15 | - [Integrate Your Module Step by Step](integration.md) 16 | -------------------------------------------------------------------------------- /src/interfaces/lookup.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | 3 | use crate::types::{config::resolver::ResolverConfig, func::AsyncIPResolveFunc}; 4 | 5 | /// IP lookup future trait implementation, lookup future is a future object that returns 6 | /// IP address of given domain address so `lookup_host_future` method should return a future 7 | /// object that resolvers IP address 8 | #[async_trait] 9 | pub trait LookUpHostFuture: Send + Sync { 10 | /// Should return `lookup_host` future object that acts according to `config.disabled` 11 | /// value. `lookup_host` future must resolve IP address by given host 12 | async fn lookup_host_future(&self) -> AsyncIPResolveFunc; 13 | /// Should return resolver configurations 14 | async fn config(&self) -> ResolverConfig; 15 | } 16 | -------------------------------------------------------------------------------- /src/cli/banner.rs: -------------------------------------------------------------------------------- 1 | /// Returns project banner as a text art with a version tag 2 | /// 3 | /// # Banner 4 | /// 5 | /// ```text 6 | /// _ 7 | /// | | 8 | /// ___ _ _| |__ ___ ___ __ _ _ __ 9 | /// / __| | | | '_ \/ __|/ __/ _` | '_ \ 10 | /// \__ \ |_| | |_) \__ \ (_| (_| | | | | 11 | /// |___/\__,_|_.__/|___/\___\__,_|_| |_| 12 | /// 13 | /// ``` 14 | pub fn banner() -> String { 15 | format!( 16 | r" 17 | _ 18 | | | 19 | ___ _ _| |__ ___ ___ __ _ _ __ 20 | / __| | | | '_ \/ __|/ __/ _` | '_ \ 21 | \__ \ |_| | |_) \__ \ (_| (_| | | | | 22 | |___/\__,_|_.__/|___/\___\__,_|_| |_| 23 | 24 | v{} 25 | ", 26 | env!("CARGO_PKG_VERSION") 27 | ) 28 | } 29 | -------------------------------------------------------------------------------- /src/utilities/serializers.rs: -------------------------------------------------------------------------------- 1 | use chrono::TimeDelta; 2 | use serde::Serializer; 3 | 4 | /// Serializer method to convert [`TimeDelta`] objects to [`i64`] seconds 5 | /// 6 | /// # Examples 7 | /// 8 | /// ``` 9 | /// use subscan::utilities::serializers::td_to_seconds; 10 | /// use chrono::TimeDelta; 11 | /// use serde_json::Serializer; 12 | /// 13 | /// let mut buffer = Vec::new(); 14 | /// let mut serializer = Serializer::new(&mut buffer); 15 | /// 16 | /// let serialized = td_to_seconds(&TimeDelta::zero(), &mut serializer); 17 | /// 18 | /// assert_eq!(String::from_utf8(buffer).unwrap(), "0"); 19 | /// ``` 20 | pub fn td_to_seconds(td: &TimeDelta, serializer: S) -> Result 21 | where 22 | S: Serializer, 23 | { 24 | serializer.serialize_i64(td.num_seconds()) 25 | } 26 | -------------------------------------------------------------------------------- /examples/crate_run_usage.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use log::LevelFilter::Debug; 4 | use subscan::{types::config::subscan::SubscanConfig, Subscan}; 5 | 6 | #[tokio::main] 7 | async fn main() { 8 | let exe = env::current_exe().unwrap(); 9 | let exe_name = exe.file_name().unwrap().to_str(); 10 | 11 | let args: Vec = env::args().collect(); 12 | let (module, target) = (&args[1], &args[2]); 13 | 14 | env_logger::builder().filter_module(exe_name.unwrap(), Debug).init(); 15 | 16 | // use default configurations 17 | let config = SubscanConfig::default(); 18 | 19 | let subscan = Subscan::from(config); 20 | let result = subscan.run(module, target).await; 21 | 22 | for item in result.items { 23 | log::debug!("{}", item.as_txt()) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | ## Contributing 3 | 4 | First off, thank you for considering contributing to Subscan! 🎉 Your support helps us build a more powerful and reliable tool 5 | 6 | ## How to Get Started 7 | 8 | To make your contribution process smooth, we've prepared detailed development documentation. You can find everything you need in the [Development](https://www.erdoganyoksul.com/subscan/development/index.html) chapter of project book 9 | 10 | ## Acknowledgements 11 | 12 | We deeply appreciate every contributor's effort and time. Whether it's fixing a bug, suggesting improvements, or adding a new feature, your contributions make Subscan better for everyone 13 | 14 | Thank you for being part of this journey! 🙌 15 | 16 | -------------------------------------------------------------------------------- /dist-workspace.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["cargo:."] 3 | 4 | # Config for 'dist' 5 | [dist] 6 | # The preferred dist version to use in CI (Cargo.toml SemVer syntax) 7 | cargo-dist-version = "0.29.0" 8 | # CI backends to support 9 | ci = "github" 10 | # The installers to generate for each app 11 | installers = ["shell", "powershell"] 12 | # Target platforms to build apps for (Rust target-triple syntax) 13 | targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] 14 | # Path that installers should place binaries in 15 | install-path = "CARGO_HOME" 16 | # Whether to install an updater program 17 | install-updater = false 18 | # Which actions to run on pull requests 19 | pr-run-mode = "upload" 20 | # Ignore out-of-date contents 21 | allow-dirty = ["ci"] 22 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | 13 | - package-ecosystem: "docker" 14 | directory: "/" 15 | schedule: 16 | interval: "weekly" 17 | 18 | - package-ecosystem: "cargo" # See documentation for possible values 19 | directory: "/" # Location of package manifests 20 | schedule: 21 | interval: "weekly" 22 | -------------------------------------------------------------------------------- /testing/testdata/txt/resolverlist.txt: -------------------------------------------------------------------------------- 1 | INVALID IPV4 2 | ------------ 3 | foo 4 | foo:bar 5 | foo:bar:baz 6 | 127.0.0.1:0000 7 | 192.168.44.10: 8 | 192.168.44.10:65537 9 | 256.256.256.256:65535 10 | 256.123.45.81:65535 11 | 123 12 | 123.1231.123.123:65 13 | 123:123:123:123 14 | 4444.44 15 | 127.0.0 16 | 17 | VALID IPV4 18 | ------------ 19 | 127.0.0.1:1 20 | 127.9.2.129:25 21 | 176.255.45.12:123 22 | 192.168.1.1:8080 23 | 10.126.125.98:4444 24 | 0.0.0.0:4444 25 | 26 | INVALID IPV6 27 | ------------ 28 | foo:123 29 | [foo]:bar 30 | [foo]:123 31 | []:123 32 | [123]: 33 | [foo.bar]:15123 34 | [abcd:1234::123::1]:4213 35 | 36 | VALID IPV6 37 | ------------ 38 | [2001:db8::1]:8080 39 | [2001:db8:85a3:8d3:1319:8a2e:370:7348]:443 40 | [abcd:ef::42:1]:8080 41 | [0:0:0:0:0:ffff:1.2.3.4]:4444 42 | [::1]:1234 43 | [::192.168.30.2]:8001 44 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/builtwith.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "KEY": { 6 | "equalTo": "builtwith-api-key" 7 | } 8 | }, 9 | "urlPath": "/builtwith" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "Results": [ 17 | { 18 | "Result": { 19 | "Paths": [ 20 | { 21 | "SubDomain": "bar" 22 | } 23 | ] 24 | } 25 | } 26 | ] 27 | }, 28 | "status": 200 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /book/src/development/components/README.md: -------------------------------------------------------------------------------- 1 | # Components 2 | 3 | This chapter provides detailed information about the functionality of the core components that make up `Subscan`. These components are reusable structures designed to simplify repetitive tasks, such as organizing HTTP requests or facilitating subdomain extraction operations. By using these components, you can streamline your workflow and avoid redundant code 4 | 5 | You can also create custom components tailored to your specific needs and integrate them into the subdomain discovery process. These components add modularity to `Subscan`, allowing it to be easily extended and customized 6 | 7 | The core components in `Subscan` are listed below. Follow the links for more details 8 | 9 | - [Requesters](requesters.md) 10 | - [Extractors](extractors.md) 11 | - [Subscan Module](module.md) 12 | -------------------------------------------------------------------------------- /src/utilities/env.rs: -------------------------------------------------------------------------------- 1 | use crate::constants::SUBSCAN_ENV_NAMESPACE; 2 | 3 | /// Formats given module name and environment variable name with [`SUBSCAN_ENV_NAMESPACE`] 4 | /// prefix, returns fully generated environment variable name 5 | /// 6 | /// # Examples 7 | /// 8 | /// ``` 9 | /// use subscan::utilities::env::format_env; 10 | /// 11 | /// #[tokio::main] 12 | /// async fn main() { 13 | /// assert_eq!(format_env("foo", "apikey"), "SUBSCAN_FOO_APIKEY"); 14 | /// assert_eq!(format_env("foo", "username"), "SUBSCAN_FOO_USERNAME"); 15 | /// assert_eq!(format_env("bar", "password"), "SUBSCAN_BAR_PASSWORD"); 16 | /// } 17 | /// ``` 18 | pub fn format_env(name: &str, env: &str) -> String { 19 | format!( 20 | "{}_{}_{}", 21 | SUBSCAN_ENV_NAMESPACE, 22 | name.to_uppercase(), 23 | env.to_uppercase(), 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /src/logger.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | 3 | use colog::format::{CologStyle, DefaultCologStyle}; 4 | use env_logger::fmt::Formatter; 5 | use log::{LevelFilter, Record}; 6 | 7 | use crate::constants::SUBSCAN_BANNER_LOG_TARGET; 8 | 9 | /// Initialize logger 10 | pub async fn init(level: Option) { 11 | let pkg_name = env!("CARGO_PKG_NAME"); 12 | let filter = level.unwrap_or(LevelFilter::Debug); 13 | 14 | env_logger::builder().filter_module(pkg_name, filter).format(formatter).init(); 15 | } 16 | 17 | // Custom formatter to avoid timestamp and log levels on banner log line 18 | fn formatter(buf: &mut Formatter, record: &Record<'_>) -> Result<(), std::io::Error> { 19 | if record.target() == SUBSCAN_BANNER_LOG_TARGET { 20 | writeln!(buf, "{}", record.args()) 21 | } else { 22 | DefaultCologStyle.format(buf, record) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/whoisxmlapi.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "apiKey": { 6 | "equalTo": "whoisxmlapi-api-key" 7 | } 8 | }, 9 | "urlPath": "/whoisxmlapi" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": { 16 | "result": { 17 | "count": 1, 18 | "records": [ 19 | { 20 | "domain": "bar.foo.com", 21 | "firstSeen": 1678185120, 22 | "lastSeen": 1678185120 23 | } 24 | ] 25 | }, 26 | "search": "foo.com" 27 | }, 28 | "status": 200 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /examples/custom_extractor.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use async_trait::async_trait; 4 | use subscan::{ 5 | enums::content::Content, 6 | interfaces::extractor::SubdomainExtractorInterface, 7 | types::core::{Result, Subdomain}, 8 | }; 9 | 10 | pub struct CustomExtractor {} 11 | 12 | #[async_trait] 13 | impl SubdomainExtractorInterface for CustomExtractor { 14 | async fn extract(&self, content: Content, _domain: &str) -> Result> { 15 | let subdomain = content.as_string().replace("-", ""); 16 | 17 | Ok([subdomain].into()) 18 | } 19 | } 20 | 21 | #[tokio::main] 22 | async fn main() { 23 | let content = Content::from("--foo.com--"); 24 | let extractor = CustomExtractor {}; 25 | let result = extractor.extract(content, "foo.com").await.unwrap(); 26 | 27 | assert_eq!(result, ["foo.com".into()].into()); 28 | } 29 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/waybackarchive/waybackarchive.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "collapse": { 6 | "equalTo": "urlkey" 7 | }, 8 | "fl": { 9 | "equalTo": "original" 10 | }, 11 | "output": { 12 | "equalTo": "txt" 13 | }, 14 | "url": { 15 | "equalTo": "*.foo.com/*" 16 | } 17 | }, 18 | "urlPath": "/waybackarchive" 19 | }, 20 | "response": { 21 | "body": "bar.foo.com\nbaz.foo.com", 22 | "headers": { 23 | "accept-ranges": "bytes", 24 | "connection": "keep-alive", 25 | "content-length": 23, 26 | "content-type": "text/html" 27 | }, 28 | "status": 200 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /book/src/user-guide/README.md: -------------------------------------------------------------------------------- 1 | # User Guide 2 | 3 | This chapter provides an overview of the basic usage of `Subscan`, designed to help end users get started quickly and effectively 4 | 5 | Here’s a quick overview of the sections included 6 | 7 | 8 | 9 | - [Quickstart](quickstart/index.html) 10 | - [Install](quickstart/install.md) 11 | - [Usage](quickstart/usage/index.html) 12 | - [CLI](quickstart/usage/cli.md) 13 | - [Docker](quickstart/usage/docker.md) 14 | - [Crate](quickstart/usage/crate.md) 15 | - [Commands](commands/index.html) 16 | - [scan](commands/scan.md) 17 | - [brute](commands/brute.md) 18 | - [module](commands/module.md) 19 | - [list](commands/module.md#list) 20 | - [get](commands/module.md#get) 21 | - [run](commands/module.md#run) 22 | - [Environments](environments.md) 23 | 24 | 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.91-slim-bookworm AS builder 2 | 3 | WORKDIR /builder 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update -y && \ 8 | apt-get install -y --no-install-recommends \ 9 | chromium \ 10 | libssl-dev \ 11 | pkg-config \ 12 | tini \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | COPY Cargo.toml . 16 | COPY Cargo.lock . 17 | COPY src src 18 | 19 | RUN cargo build --release 20 | 21 | # hadolint ignore=DL3007 22 | FROM gcr.io/distroless/cc-debian12:latest 23 | 24 | ENV SUBSCAN_CHROME_PATH=/usr/lib/chromium/chromium 25 | 26 | # Copy libs from builder 27 | COPY --from=builder /usr/lib /usr/lib 28 | COPY --from=builder /usr/share /usr/share 29 | # Copy required binaries 30 | COPY --from=builder /usr/bin/tini /bin/tini 31 | COPY --from=builder /builder/target/release/subscan /bin/subscan 32 | 33 | WORKDIR /data 34 | 35 | ENTRYPOINT ["tini", "--", "subscan"] 36 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/crtsh.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/crtsh" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": [ 11 | { 12 | "common_name": "bar.foo.com", 13 | "entry_timestamp": "2024-06-14T07:17:12.05", 14 | "id": 13356418102, 15 | "issuer_ca_id": 247125, 16 | "issuer_name": "C=US, O=Foo, CN=Foo RSA 2048 M02", 17 | "name_value": "bar.foo.com", 18 | "not_after": "2025-07-13T23:59:59", 19 | "not_before": "2024-06-14T00:00:00", 20 | "result_count": 1, 21 | "serial_number": "0140a766bf10b20714379fd5ddce96f2" 22 | } 23 | ], 24 | "status": 200 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/cli/mod.rs: -------------------------------------------------------------------------------- 1 | /// CLI banner 2 | pub mod banner; 3 | /// List of CLI commands 4 | pub mod commands; 5 | 6 | use banner::banner; 7 | use clap::Parser; 8 | use clap_verbosity_flag::{DebugLevel, Verbosity}; 9 | 10 | use crate::{cli::commands::Commands, constants::SUBSCAN_BANNER_LOG_TARGET, logger}; 11 | 12 | /// Data structure for CLI, stores configurations to be used on run-time 13 | #[derive(Clone, Debug, Parser)] 14 | #[command(version, about = banner(), long_about = banner())] 15 | pub struct Cli { 16 | #[command(subcommand)] 17 | pub command: Commands, 18 | #[command(flatten)] 19 | pub verbose: Verbosity, 20 | } 21 | 22 | impl Cli { 23 | pub async fn init(&self) { 24 | logger::init(Some(self.verbose.log_level_filter())).await; 25 | } 26 | 27 | pub async fn banner(&self) { 28 | log::debug!(target: SUBSCAN_BANNER_LOG_TARGET, "{}", banner()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/utilities/regex.rs: -------------------------------------------------------------------------------- 1 | use core::result::Result; 2 | 3 | use regex::{Error, Regex}; 4 | 5 | /// Helper function that generates dynamically regex statement 6 | /// by given domain address to parse subdomains 7 | /// 8 | /// # Examples 9 | /// 10 | /// ``` 11 | /// use subscan::utilities::regex::generate_subdomain_regex; 12 | /// 13 | /// let regex_stmt = generate_subdomain_regex("foo.com").unwrap(); 14 | /// 15 | /// assert_eq!(regex_stmt.as_str(), "(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+(foo\\.com)"); 16 | /// 17 | /// assert!(regex_stmt.find("bar.foo.com").is_some()); 18 | /// assert!(regex_stmt.find("foo").is_none()); 19 | /// ``` 20 | pub fn generate_subdomain_regex(domain: &str) -> Result { 21 | let formatted = format!( 22 | r"(?:[a-z0-9](?:[a-z0-9-]{{0,61}}[a-z0-9])?\.)+({domain})", 23 | domain = domain.replace('.', r"\.") 24 | ); 25 | 26 | Regex::new(&formatted) 27 | } 28 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/waybackarchive/waybackarchive-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "collapse": { 6 | "equalTo": "urlkey" 7 | }, 8 | "fl": { 9 | "equalTo": "original" 10 | }, 11 | "output": { 12 | "equalTo": "txt" 13 | }, 14 | "url": { 15 | "equalTo": "*.foo.com/*" 16 | } 17 | }, 18 | "urlPath": "/waybackarchive-delayed" 19 | }, 20 | "response": { 21 | "body": "bar.foo.com\nbaz.foo.com", 22 | "fixedDelayMilliseconds": 1000, 23 | "headers": { 24 | "accept-ranges": "bytes", 25 | "connection": "keep-alive", 26 | "content-length": 23, 27 | "content-type": "text/html" 28 | }, 29 | "status": 200 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tests/components/resolver_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{interfaces::lookup::LookUpHostFuture, types::config::resolver::ResolverConfig}; 2 | 3 | use crate::common::{ 4 | constants::{LOCAL_HOST, TEST_DOMAIN}, 5 | mock::resolver::MockResolver, 6 | }; 7 | 8 | #[tokio::test] 9 | async fn lookup_host_future_test_with_returns_none() { 10 | let rconfig = ResolverConfig { 11 | disabled: true, 12 | ..Default::default() 13 | }; 14 | let resolver = MockResolver::boxed(rconfig); 15 | let lookup_host = resolver.lookup_host_future().await; 16 | 17 | assert!(lookup_host(TEST_DOMAIN.into()).await.is_none()); 18 | } 19 | 20 | #[tokio::test] 21 | async fn lookup_host_future_test_with_returns_ip() { 22 | let resolver = MockResolver::default_boxed(); 23 | let lookup_host = resolver.lookup_host_future().await; 24 | let ip = lookup_host(TEST_DOMAIN.into()).await.unwrap(); 25 | 26 | assert_eq!(ip.to_string(), LOCAL_HOST); 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | env: 12 | RUSTFLAGS: "-Dwarnings" 13 | CARGO_TERM_COLOR: "always" 14 | 15 | jobs: 16 | rust-cargo-test: 17 | strategy: 18 | matrix: 19 | os: [ubuntu-latest, windows-latest, macos-latest] 20 | 21 | runs-on: ${{ matrix.os }} 22 | steps: 23 | - uses: actions/checkout@v6 24 | - uses: dtolnay/rust-toolchain@stable 25 | - uses: Swatinem/rust-cache@v2 26 | - uses: taiki-e/install-action@nextest 27 | - uses: browser-actions/setup-chrome@v2 28 | id: setup-chrome 29 | with: 30 | chrome-version: latest 31 | 32 | - name: Run nextest 33 | run: make nextest 34 | env: 35 | SUBSCAN_CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} 36 | 37 | - name: Run doc-test 38 | run: make doc-test 39 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Rust specific 2 | target/ 3 | **/*.rs.bk 4 | *.pdb 5 | 6 | # Cache directories 7 | .cargo-cache/ 8 | .cargo/ 9 | 10 | # Tests and examples 11 | tests/ 12 | examples/ 13 | benches/ 14 | 15 | # Documentation 16 | docs/ 17 | *.md 18 | README* 19 | 20 | # Docker-related files 21 | Dockerfile* 22 | .dockerignore 23 | docker-compose* 24 | 25 | # Git-related files 26 | .git/ 27 | .gitignore 28 | 29 | # Logs, databases and temporary files 30 | *.sqlite 31 | *.db 32 | *.log 33 | *.tmp 34 | *.out 35 | 36 | # Temporary folders 37 | tmp/ 38 | temp/ 39 | 40 | # Project specific backups 41 | backup/ 42 | *.bak 43 | *~ 44 | 45 | # Envs 46 | .env 47 | .env.* 48 | *.env 49 | 50 | # IDE/Editor-specific files 51 | *.rs.bk 52 | *.swp 53 | *.swo 54 | *.swn 55 | *.bak 56 | *.tmp 57 | *.iml 58 | .project 59 | .classpath 60 | .idea/ 61 | .vscode/ 62 | .settings/ 63 | 64 | # OS-generated files 65 | .DS_Store 66 | Thumbs.db 67 | .Spotlight-V100 68 | ehthumbs.db 69 | .Trashes 70 | ._* 71 | -------------------------------------------------------------------------------- /src/cli/commands/module/mod.rs: -------------------------------------------------------------------------------- 1 | /// Get command to fetch any module by name 2 | pub mod get; 3 | /// List command to list modules with details 4 | pub mod list; 5 | /// Run command to start any module 6 | pub mod run; 7 | 8 | use clap::{Args, Subcommand}; 9 | 10 | use crate::cli::commands::module::{ 11 | get::ModuleGetSubCommandArgs, list::ModuleListSubCommandArgs, run::ModuleRunSubCommandArgs, 12 | }; 13 | 14 | /// List of subcommands on module command 15 | #[derive(Debug, Clone, Subcommand)] 16 | pub enum ModuleSubCommands { 17 | /// Run a single module by name 18 | Run(ModuleRunSubCommandArgs), 19 | /// List all registered modules with their details 20 | List(ModuleListSubCommandArgs), 21 | /// Get a single module details 22 | Get(ModuleGetSubCommandArgs), 23 | } 24 | 25 | /// Module subcommand container 26 | #[derive(Args, Clone, Debug)] 27 | pub struct ModuleCommandArgs { 28 | #[command(subcommand)] 29 | pub command: ModuleSubCommands, 30 | } 31 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/github/github-code-search-no-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "token github-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "order": { 11 | "equalTo": "asc" 12 | }, 13 | "per_page": { 14 | "equalTo": "100" 15 | }, 16 | "q": { 17 | "equalTo": "foo.com" 18 | }, 19 | "sort": { 20 | "equalTo": "created" 21 | } 22 | }, 23 | "urlPath": "/github-code-search-no-data" 24 | }, 25 | "response": { 26 | "headers": { 27 | "content-type": "text/html" 28 | }, 29 | "jsonBody": {}, 30 | "status": 200, 31 | "transformers": [ 32 | "response-template" 33 | ] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/interfaces/extractor.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use async_trait::async_trait; 4 | use enum_dispatch::enum_dispatch; 5 | 6 | use crate::{ 7 | enums::{content::Content, dispatchers::SubdomainExtractorDispatcher}, 8 | extractors::{html::HTMLExtractor, json::JSONExtractor, regex::RegexExtractor}, 9 | types::core::{Result, Subdomain}, 10 | }; 11 | 12 | /// Extractor trait definition to implement subdomain extractors 13 | /// 14 | /// All subdomain extractors that implemented in the future must be compatible with this 15 | /// trait. Basically it has single `extract` method like a `main` method. It should 16 | /// extract subdomain addresses and return them from given [`String`] content 17 | #[async_trait] 18 | #[enum_dispatch] 19 | pub trait SubdomainExtractorInterface: Send + Sync { 20 | /// Generic extract method, it should extract subdomain addresses 21 | /// from given [`Content`] 22 | async fn extract(&self, content: Content, domain: &str) -> Result>; 23 | } 24 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/github/github-code-search-delayed.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "token github-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "order": { 11 | "equalTo": "asc" 12 | }, 13 | "per_page": { 14 | "equalTo": "100" 15 | }, 16 | "q": { 17 | "equalTo": "foo.com" 18 | }, 19 | "sort": { 20 | "equalTo": "created" 21 | } 22 | }, 23 | "urlPath": "/github-code-search-delayed" 24 | }, 25 | "response": { 26 | "fixedDelayMilliseconds": 1000, 27 | "headers": { 28 | "content-type": "text/html" 29 | }, 30 | "jsonBody": {}, 31 | "status": 200, 32 | "transformers": [ 33 | "response-template" 34 | ] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /examples/crate_brute_usage.rs: -------------------------------------------------------------------------------- 1 | use std::{env, io::Write}; 2 | 3 | use log::LevelFilter::Debug; 4 | use subscan::{types::config::subscan::SubscanConfig, Subscan}; 5 | use tempfile::NamedTempFile; 6 | 7 | #[tokio::main] 8 | async fn main() { 9 | let exe = env::current_exe().unwrap(); 10 | let exe_name = exe.file_name().unwrap().to_str(); 11 | let args: Vec = env::args().collect(); 12 | 13 | env_logger::builder().filter_module(exe_name.unwrap(), Debug).init(); 14 | 15 | let mut wordlist = NamedTempFile::new().unwrap(); 16 | 17 | writeln!(wordlist, "api").unwrap(); 18 | writeln!(wordlist, "app").unwrap(); 19 | writeln!(wordlist, "test").unwrap(); 20 | 21 | let config = SubscanConfig { 22 | wordlist: Some(wordlist.path().to_path_buf()), 23 | ..Default::default() 24 | }; 25 | 26 | let subscan = Subscan::from(config); 27 | let result = subscan.brute(&args[1]).await; 28 | 29 | for item in result.items { 30 | log::debug!("{}", item.as_txt()) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/types/result/pool.rs: -------------------------------------------------------------------------------- 1 | use super::{item::SubscanResultItem, statistics::SubscanModuleStatistic}; 2 | use crate::types::result::{item::SubscanResultItems, statistics::SubscanResultStatistics}; 3 | 4 | /// Stores [`SubscanModulePool`](crate::pools::module::SubscanModulePool) results 5 | #[derive(Clone, Debug, Default)] 6 | pub struct PoolResult { 7 | /// Pool statistics, includes each module statistics 8 | /// and IP resolver statistics 9 | pub statistics: SubscanResultStatistics, 10 | /// Subdomains that have been discovered 11 | pub items: SubscanResultItems, 12 | } 13 | 14 | impl PoolResult { 15 | pub async fn insert(&mut self, module: &str, subdomain: SubscanResultItem) -> bool { 16 | let defaults = SubscanModuleStatistic::default(); 17 | let inserted = self.items.insert(subdomain); 18 | 19 | let stats = self.statistics.entry(module.to_owned()).or_insert(defaults); 20 | 21 | if inserted { 22 | stats.count += 1; 23 | } 24 | 25 | inserted 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/netlas/with-count/netlas-domains-download.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "bodyPatterns": [ 4 | { 5 | "equalToJson": { 6 | "fields": [ 7 | "*" 8 | ], 9 | "q": "domain:(domain:*.foo.com AND NOT domain:foo.com)", 10 | "size": 1, 11 | "source_type": "include" 12 | } 13 | } 14 | ], 15 | "headers": { 16 | "X-API-Key": { 17 | "equalTo": "netlas-api-key" 18 | } 19 | }, 20 | "method": "POST", 21 | "urlPath": "/api/domains/download/" 22 | }, 23 | "response": { 24 | "headers": { 25 | "content-type": "application/json" 26 | }, 27 | "jsonBody": [ 28 | { 29 | "data": { 30 | "domain": "bar.foo.com" 31 | } 32 | } 33 | ], 34 | "status": 200 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /examples/custom_requester.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use reqwest::Url; 3 | use subscan::{ 4 | enums::content::Content, 5 | interfaces::requester::RequesterInterface, 6 | types::{config::requester::RequesterConfig, core::Result}, 7 | }; 8 | 9 | pub struct CustomRequester { 10 | config: RequesterConfig, 11 | } 12 | 13 | #[async_trait] 14 | impl RequesterInterface for CustomRequester { 15 | async fn config(&mut self) -> &mut RequesterConfig { 16 | &mut self.config 17 | } 18 | 19 | async fn configure(&mut self, config: RequesterConfig) { 20 | self.config = config; 21 | } 22 | 23 | async fn get_content(&self, _url: Url) -> Result { 24 | Ok(Content::Empty) 25 | } 26 | } 27 | 28 | #[tokio::main] 29 | async fn main() { 30 | let url = Url::parse("https://example.com").unwrap(); 31 | let requester = CustomRequester { 32 | config: RequesterConfig::default(), 33 | }; 34 | 35 | let content = requester.get_content(url).await.unwrap(); 36 | 37 | assert_eq!(content.as_string(), ""); 38 | } 39 | -------------------------------------------------------------------------------- /book/src/user-guide/quickstart/install.md: -------------------------------------------------------------------------------- 1 | # Install 2 | 3 | There are several ways to install `Subscan`, depending on your preferences. You can install it via [Cargo](#install-with-cargo) (Rust's package manager), use [Docker](#pull-docker-image) for containerized environments, or download prebuilt [cross-platform binaries](#download-prebuilt-binaries). Choose the method that works best for your setup 4 | 5 | ## Install With Cargo 6 | 7 | 🦀 Install the subscan tool using Cargo, Rust's package manager. Make sure you have [Rust](https://www.rust-lang.org/) installed on your system. Then, run: 8 | 9 | ```bash 10 | ~$ cargo install subscan 11 | ``` 12 | 13 | ## Pull Docker Image 14 | 15 | 🐳 For containerized usage, you can pull the subscan Docker image directly from [Docker Hub](https://hub.docker.com/) 16 | 17 | ```bash 18 | ~$ docker pull eredotpkfr/subscan:latest 19 | ``` 20 | 21 | ## Download Prebuilt Binaries 22 | 23 | 📦 Prebuilt cross-platform binaries are available on the [releases page](https://github.com/eredotpkfr/subscan/releases). Download the one compatible with your operating system 24 | -------------------------------------------------------------------------------- /tests/components/modules/engines/duckduckgo_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{ 2 | enums::dispatchers::SubscanModuleDispatcher, modules::engines::duckduckgo::DuckDuckGo, 3 | requesters::client::HTTPClient, types::result::status::SubscanModuleStatus, 4 | }; 5 | use tokio::sync::Mutex; 6 | 7 | use crate::common::{ 8 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN}, 9 | mock::funcs, 10 | utils, 11 | }; 12 | 13 | #[tokio::test] 14 | #[stubr::mock("module/engines/duckduckgo.json")] 15 | async fn run_test() { 16 | let mut duckduckgo = DuckDuckGo::dispatcher(); 17 | let new_requester = HTTPClient::default(); 18 | 19 | funcs::wrap_module_url(&mut duckduckgo, &stubr.uri()); 20 | 21 | if let SubscanModuleDispatcher::GenericSearchEngineModule(ref mut duckduckgo) = duckduckgo { 22 | duckduckgo.components.requester = Mutex::new(new_requester.into()); 23 | } 24 | 25 | let (results, status) = utils::run_module(duckduckgo, TEST_DOMAIN).await; 26 | 27 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 28 | assert_eq!(status, SubscanModuleStatus::Finished); 29 | } 30 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/certspotter.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "certspotter-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "urlPath": "/certspotter" 10 | }, 11 | "response": { 12 | "headers": { 13 | "content-type": "application/json" 14 | }, 15 | "jsonBody": [ 16 | { 17 | "cert_sha256": "360c41cbc12e1006f817f61819be80a892af7acf4cff89b43e90d0892d1ff221", 18 | "dns_names": [ 19 | "bar.foo.com" 20 | ], 21 | "id": "8304288493", 22 | "not_after": "2024-12-30T12:46:39Z", 23 | "not_before": "2024-10-01T12:46:40Z", 24 | "pubkey_sha256": "abaddb27dfb94543a30d58656e2b0187889c26e5fd47c54cac1f568ecd99b3da", 25 | "revoked": false, 26 | "tbs_sha256": "79e9457e879c77d5a2306f0378456c0ef84125fbb593755886215fcb5b1f8c36c" 27 | } 28 | ], 29 | "status": 200 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Erdoğan YOKSUL 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/crate_scan_usage.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use log::LevelFilter::Debug; 4 | use subscan::{ 5 | enums::cache::CacheFilter::FilterByName, 6 | types::{config::subscan::SubscanConfig, filters::ModuleNameFilter}, 7 | Subscan, 8 | }; 9 | 10 | #[tokio::main] 11 | async fn main() { 12 | let exe = env::current_exe().unwrap(); 13 | let exe_name = exe.file_name().unwrap().to_str(); 14 | let args: Vec = env::args().collect(); 15 | 16 | env_logger::builder().filter_module(exe_name.unwrap(), Debug).init(); 17 | 18 | // filter modules by name, only runs google and alienvault modules 19 | let filter = ModuleNameFilter { 20 | modules: vec!["google".into(), "alienvault".into()], 21 | skips: vec![], 22 | }; 23 | 24 | // set module conccurrency to 1 25 | let config = SubscanConfig { 26 | concurrency: 1, 27 | filter: FilterByName(filter), 28 | ..Default::default() 29 | }; 30 | 31 | let subscan = Subscan::from(config); 32 | let result = subscan.scan(&args[1]).await; 33 | 34 | for item in result.items { 35 | log::debug!("{}", item.as_txt()) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/mdbook.yml: -------------------------------------------------------------------------------- 1 | name: Github Pages 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | mdbook-tests: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v6 16 | 17 | - name: Setup mdBook 18 | uses: peaceiris/actions-mdbook@v2 19 | with: 20 | mdbook-version: 'latest' 21 | 22 | - name: Run mdBook tests 23 | run: make book-test 24 | 25 | mdbook-deploy: 26 | runs-on: ubuntu-latest 27 | permissions: 28 | contents: write 29 | needs: mdbook-tests 30 | steps: 31 | - uses: actions/checkout@v6 32 | 33 | - name: Setup mdBook 34 | uses: peaceiris/actions-mdbook@v2 35 | with: 36 | mdbook-version: 'latest' 37 | 38 | - name: Build mdBook 39 | run: make book-build 40 | 41 | - name: Deploy to GitHub Pages 42 | uses: peaceiris/actions-gh-pages@v4 43 | if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} 44 | with: 45 | github_token: ${{ secrets.GITHUB_TOKEN }} 46 | publish_dir: book/book 47 | -------------------------------------------------------------------------------- /tests/components/extractors/regex_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use subscan::{ 4 | enums::content::Content, extractors::regex::RegexExtractor, 5 | interfaces::extractor::SubdomainExtractorInterface, 6 | }; 7 | 8 | use crate::common::constants::{TEST_BAR_SUBDOMAIN, TEST_BAZ_SUBDOMAIN, TEST_DOMAIN}; 9 | 10 | #[tokio::test] 11 | async fn extract_one_test() { 12 | let extractor = RegexExtractor::default(); 13 | 14 | let matches = String::from(TEST_BAR_SUBDOMAIN); 15 | let no_matches = String::from("foobarbaz"); 16 | 17 | assert!(extractor.extract_one(matches, TEST_DOMAIN).is_some()); 18 | assert!(extractor.extract_one(no_matches, TEST_DOMAIN).is_none()); 19 | } 20 | 21 | #[tokio::test] 22 | async fn extract_test() { 23 | let content = Content::from("bar.foo.com\nbaz.foo.com"); 24 | 25 | let extractor = RegexExtractor::default(); 26 | let result = extractor.extract(content, TEST_DOMAIN).await; 27 | 28 | let expected = BTreeSet::from([ 29 | TEST_BAR_SUBDOMAIN.to_string(), 30 | TEST_BAZ_SUBDOMAIN.to_string(), 31 | ]); 32 | 33 | assert!(result.is_ok()); 34 | assert_eq!(result.unwrap(), expected); 35 | } 36 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/censys.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "censys-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "cursor": { 11 | "absent": true 12 | } 13 | }, 14 | "urlPath": "/censys" 15 | }, 16 | "response": { 17 | "headers": { 18 | "content-type": "application/json" 19 | }, 20 | "jsonBody": { 21 | "result": { 22 | "hits": [ 23 | { 24 | "names": [ 25 | "bar.foo.com", 26 | "*.bar.foo.com" 27 | ] 28 | }, 29 | { 30 | "names": { 31 | "foo": "bar" 32 | } 33 | } 34 | ], 35 | "links": { 36 | "next": "cursor" 37 | } 38 | } 39 | }, 40 | "status": 200 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/interfaces/requester.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use enum_dispatch::enum_dispatch; 3 | use reqwest::Url; 4 | 5 | use crate::{ 6 | enums::{content::Content, dispatchers::RequesterDispatcher}, 7 | requesters::{chrome::ChromeBrowser, client::HTTPClient}, 8 | types::{config::requester::RequesterConfig, core::Result}, 9 | }; 10 | 11 | /// Generic HTTP client trait definition to implement different HTTP requester objects 12 | /// with a single interface compatible 13 | /// 14 | /// Other requesters that will be implemented in the future must conform to this interface. 15 | /// Mostly uses to get string content from any URL with a single stupid `get_content` method 16 | #[async_trait] 17 | #[enum_dispatch] 18 | pub trait RequesterInterface: Sync + Send { 19 | /// Returns requester configurations as a [`RequesterConfig`] object 20 | async fn config(&mut self) -> &mut RequesterConfig; 21 | /// Configure current requester object by using new [`RequesterConfig`] object 22 | async fn configure(&mut self, config: RequesterConfig); 23 | /// HTTP GET method implementation to fetch HTML content from given source [`Url`] 24 | async fn get_content(&self, url: Url) -> Result; 25 | } 26 | -------------------------------------------------------------------------------- /src/types/result/metadata.rs: -------------------------------------------------------------------------------- 1 | use chrono::{serde::ts_seconds, DateTime, TimeDelta, Utc}; 2 | use serde::Serialize; 3 | 4 | use crate::utilities::serializers::td_to_seconds; 5 | 6 | /// [`SubscanResult`](crate::types::result::subscan::SubscanResult) metadata struct definition 7 | #[derive(Clone, Debug, Serialize)] 8 | pub struct SubscanResultMetadata { 9 | pub target: String, 10 | #[serde(with = "ts_seconds")] 11 | pub started_at: DateTime, 12 | #[serde(with = "ts_seconds")] 13 | pub finished_at: DateTime, 14 | #[serde(serialize_with = "td_to_seconds")] 15 | pub elapsed: TimeDelta, 16 | } 17 | 18 | impl Default for SubscanResultMetadata { 19 | fn default() -> Self { 20 | Self { 21 | target: String::new(), 22 | started_at: Utc::now(), 23 | finished_at: Utc::now(), 24 | elapsed: TimeDelta::zero(), 25 | } 26 | } 27 | } 28 | 29 | impl From<&str> for SubscanResultMetadata { 30 | fn from(target: &str) -> Self { 31 | Self { 32 | target: target.to_string(), 33 | started_at: Utc::now(), 34 | finished_at: Utc::now(), 35 | elapsed: TimeDelta::zero(), 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/dnsrepo_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{ 2 | enums::content::Content, 3 | modules::integrations::dnsrepo::{DnsRepo, DNSREPO_URL}, 4 | types::result::status::SubscanModuleStatus, 5 | }; 6 | 7 | use crate::common::{ 8 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 9 | mock::funcs, 10 | utils, 11 | }; 12 | 13 | #[tokio::test] 14 | #[stubr::mock("module/integrations/dnsrepo.json")] 15 | async fn run_test() { 16 | let mut dnsrepo = DnsRepo::dispatcher(); 17 | 18 | funcs::wrap_module_url(&mut dnsrepo, &stubr.path("/dnsrepo")); 19 | 20 | let (results, status) = utils::run_module(dnsrepo, TEST_DOMAIN).await; 21 | 22 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 23 | assert_eq!(status, SubscanModuleStatus::Finished); 24 | } 25 | 26 | #[tokio::test] 27 | async fn get_query_url_test() { 28 | let url = DnsRepo::get_query_url(TEST_DOMAIN); 29 | let expected = format!("{DNSREPO_URL}/?search={TEST_DOMAIN}"); 30 | 31 | assert_eq!(url, expected); 32 | } 33 | 34 | #[tokio::test] 35 | async fn get_next_url_test() { 36 | let url = TEST_URL.parse().unwrap(); 37 | let next = DnsRepo::get_next_url(url, Content::Empty); 38 | 39 | assert!(next.is_none()); 40 | } 41 | -------------------------------------------------------------------------------- /src/enums/cache.rs: -------------------------------------------------------------------------------- 1 | use crate::types::filters::ModuleNameFilter; 2 | 3 | /// Cache filter variants, it allows to run filter on module cache 4 | #[derive(Clone, Debug, Default, PartialEq)] 5 | pub enum CacheFilter { 6 | /// Do nothing to eliminate modules from cache 7 | #[default] 8 | NoFilter, 9 | /// Filter modules by their names 10 | FilterByName(ModuleNameFilter), 11 | } 12 | 13 | impl CacheFilter { 14 | /// Check module name is filtered or non-filtered by filter type 15 | /// 16 | /// # Examples 17 | /// 18 | /// ``` 19 | /// use subscan::enums::cache::CacheFilter; 20 | /// use subscan::types::filters::ModuleNameFilter; 21 | /// 22 | /// #[tokio::main] 23 | /// async fn main() { 24 | /// let filter: ModuleNameFilter = (vec![], vec![]).into(); 25 | /// 26 | /// assert!(!CacheFilter::NoFilter.is_filtered("foo").await); 27 | /// assert!(!CacheFilter::FilterByName(filter).is_filtered("foo").await); 28 | /// } 29 | /// ``` 30 | pub async fn is_filtered(&self, name: &str) -> bool { 31 | match self { 32 | CacheFilter::NoFilter => false, 33 | CacheFilter::FilterByName(filter) => filter.is_filtered(name).await, 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/utilities/http.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Url; 2 | 3 | /// Update query params without remove old query params. If the 4 | /// given parameter name non-exists it will append end of the 5 | /// query otherwise it's value will be updated 6 | /// 7 | /// # Examples 8 | /// 9 | /// ``` 10 | /// use subscan::utilities::http::update_url_query; 11 | /// use reqwest::Url; 12 | /// 13 | /// let mut url: Url = "https://foo.com".parse().unwrap(); 14 | /// 15 | /// update_url_query(&mut url, "a".into(), "b".into()); 16 | /// assert_eq!(url.to_string(), "https://foo.com/?a=b"); 17 | /// 18 | /// // does not override old `a` parameter 19 | /// update_url_query(&mut url, "x".into(), "y".into()); 20 | /// assert_eq!(url.to_string(), "https://foo.com/?a=b&x=y"); 21 | /// 22 | /// update_url_query(&mut url, "a".into(), "c".into()); 23 | /// assert_eq!(url.to_string(), "https://foo.com/?x=y&a=c"); 24 | /// ``` 25 | pub fn update_url_query(url: &mut Url, name: &str, value: &str) { 26 | let binding = url.clone(); 27 | let pairs = binding.query_pairs(); 28 | let old = pairs.filter(|item| item.0.to_lowercase() != name.to_lowercase()); 29 | 30 | url.query_pairs_mut() 31 | .clear() 32 | .extend_pairs(old) 33 | .append_pair(name, value) 34 | .finish(); 35 | } 36 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/github/github-code-search-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "headers": { 4 | "Authorization": { 5 | "equalTo": "token github-api-key" 6 | } 7 | }, 8 | "method": "GET", 9 | "queryParameters": { 10 | "order": { 11 | "equalTo": "asc" 12 | }, 13 | "per_page": { 14 | "equalTo": "100" 15 | }, 16 | "q": { 17 | "equalTo": "foo.com" 18 | }, 19 | "sort": { 20 | "equalTo": "created" 21 | } 22 | }, 23 | "urlPath": "/github-code-search" 24 | }, 25 | "response": { 26 | "headers": { 27 | "content-type": "text/html" 28 | }, 29 | "jsonBody": { 30 | "items": [ 31 | { 32 | "html_url": "http://127.0.0.1:{{port}}/{{request.pathSegments.[0]}}/results" 33 | }, 34 | { 35 | "no_html_url": "foo" 36 | } 37 | ] 38 | }, 39 | "status": 200, 40 | "transformers": [ 41 | "response-template" 42 | ] 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/digitorus_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{ 2 | enums::content::Content, 3 | modules::integrations::digitorus::{Digitorus, DIGITORUS_URL}, 4 | types::result::status::SubscanModuleStatus, 5 | }; 6 | 7 | use crate::common::{ 8 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 9 | mock::funcs, 10 | utils, 11 | }; 12 | 13 | #[tokio::test] 14 | #[stubr::mock("module/integrations/digitorus.json")] 15 | async fn run_test() { 16 | let mut digitorus = Digitorus::dispatcher(); 17 | 18 | funcs::wrap_module_url(&mut digitorus, &stubr.path("/digitorus")); 19 | 20 | let (results, status) = utils::run_module(digitorus, TEST_DOMAIN).await; 21 | 22 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 23 | assert_eq!(status, SubscanModuleStatus::Finished); 24 | } 25 | 26 | #[tokio::test] 27 | async fn get_query_url_test() { 28 | let url = Digitorus::get_query_url(TEST_DOMAIN); 29 | let expected = format!("{DIGITORUS_URL}/{TEST_DOMAIN}"); 30 | 31 | assert_eq!(url, expected); 32 | } 33 | 34 | #[tokio::test] 35 | async fn get_next_url_test() { 36 | let url = TEST_URL.parse().unwrap(); 37 | let next = Digitorus::get_next_url(url, Content::Empty); 38 | 39 | assert!(next.is_none()); 40 | } 41 | -------------------------------------------------------------------------------- /src/enums/output.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | 3 | use chrono::Utc; 4 | use clap::ValueEnum; 5 | use derive_more::Display; 6 | 7 | use crate::types::result::output::OutputFile; 8 | 9 | /// Supported output formats for reporting scan results 10 | #[derive(Copy, Clone, Debug, Default, Display, Eq, Ord, PartialEq, PartialOrd, ValueEnum)] 11 | pub enum OutputFormat { 12 | #[display("txt")] 13 | TXT, 14 | #[display("csv")] 15 | CSV, 16 | #[default] 17 | #[display("json")] 18 | JSON, 19 | #[display("html")] 20 | HTML, 21 | } 22 | 23 | impl OutputFormat { 24 | pub async fn get_output_file(&self, domain: &str) -> OutputFile { 25 | let file_name = self.get_output_file_name(domain); 26 | let file = File::create(&file_name).unwrap(); 27 | 28 | (file_name, file).into() 29 | } 30 | 31 | fn get_output_file_name(&self, domain: &str) -> String { 32 | let now = Utc::now().timestamp(); 33 | 34 | match self { 35 | OutputFormat::TXT => format!("{domain}.{now}.{self}"), 36 | OutputFormat::CSV => format!("{domain}.{now}.{self}"), 37 | OutputFormat::JSON => format!("{domain}.{now}.{self}"), 38 | OutputFormat::HTML => format!("{domain}.{now}.{self}"), 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/sitedossier_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{ 2 | enums::content::Content, 3 | modules::integrations::sitedossier::{Sitedossier, SITEDOSSIER_URL}, 4 | types::result::status::SubscanModuleStatus, 5 | }; 6 | 7 | use crate::common::{ 8 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 9 | mock::funcs, 10 | utils, 11 | }; 12 | 13 | #[tokio::test] 14 | #[stubr::mock("module/integrations/sitedossier.json")] 15 | async fn run_test() { 16 | let mut sitedossier = Sitedossier::dispatcher(); 17 | 18 | funcs::wrap_module_url(&mut sitedossier, &stubr.path("/sitedossier")); 19 | 20 | let (results, status) = utils::run_module(sitedossier, TEST_DOMAIN).await; 21 | 22 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 23 | assert_eq!(status, SubscanModuleStatus::Finished); 24 | } 25 | 26 | #[tokio::test] 27 | async fn get_query_url_test() { 28 | let url = Sitedossier::get_query_url(TEST_DOMAIN); 29 | let expected = format!("{SITEDOSSIER_URL}/{TEST_DOMAIN}"); 30 | 31 | assert_eq!(url, expected); 32 | } 33 | 34 | #[tokio::test] 35 | async fn get_next_url_test() { 36 | let url = TEST_URL.parse().unwrap(); 37 | let next = Sitedossier::get_next_url(url, Content::Empty); 38 | 39 | assert!(next.is_none()); 40 | } 41 | -------------------------------------------------------------------------------- /tests/components/extractors/html_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use subscan::{ 4 | extractors::html::HTMLExtractor, interfaces::extractor::SubdomainExtractorInterface, 5 | }; 6 | 7 | use crate::common::{ 8 | constants::{TEST_BAR_SUBDOMAIN, TEST_BAZ_SUBDOMAIN, TEST_DOMAIN}, 9 | utils::read_testdata, 10 | }; 11 | 12 | #[tokio::test] 13 | async fn extract_without_removes() { 14 | let html = read_testdata("html/subdomains.html"); 15 | 16 | let selector = String::from("article > div > a > span:first-child"); 17 | let extractor = HTMLExtractor::new(selector, vec![]); 18 | let result = extractor.extract(html, TEST_DOMAIN).await; 19 | 20 | assert_eq!(result.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 21 | } 22 | 23 | #[tokio::test] 24 | async fn extract_with_removes() { 25 | let html = read_testdata("html/subdomains-with-removes.html"); 26 | 27 | let selector = String::from("article > div > a > span"); 28 | let extractor = HTMLExtractor::new(selector, vec!["
".to_string()]); 29 | let result = extractor.extract(html, TEST_DOMAIN).await; 30 | 31 | let expected = BTreeSet::from([ 32 | TEST_BAR_SUBDOMAIN.to_string(), 33 | TEST_BAZ_SUBDOMAIN.to_string(), 34 | ]); 35 | 36 | assert_eq!(result.unwrap(), expected); 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linters 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | env: 9 | RUSTDOCFLAGS: "-Dwarnings" 10 | RUSTFLAGS: "-Dwarnings" 11 | CARGO_TERM_COLOR: "always" 12 | 13 | jobs: 14 | rust-lint-checks: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v6 18 | - uses: dtolnay/rust-toolchain@nightly 19 | with: 20 | components: rustfmt, clippy 21 | - uses: Swatinem/rust-cache@v2 22 | 23 | - name: Run rustfmt 24 | run: make rustfmt-check 25 | 26 | - name: Run clippy 27 | run: make clippy 28 | 29 | rust-lint-docs: 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v6 33 | - uses: dtolnay/rust-toolchain@nightly 34 | - uses: Swatinem/rust-cache@v2 35 | - uses: dtolnay/install@cargo-docs-rs 36 | 37 | - name: Run doc-rs 38 | run: make doc-rs 39 | 40 | hadolint: 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v6 44 | - uses: hadolint/hadolint-action@v3.3.0 45 | with: 46 | dockerfile: Dockerfile 47 | 48 | typos: 49 | runs-on: ubuntu-latest 50 | steps: 51 | - uses: actions/checkout@v6 52 | - uses: crate-ci/typos@master 53 | -------------------------------------------------------------------------------- /tests/cache_test.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use subscan::{ 4 | cache::CacheManager, 5 | interfaces::{module::SubscanModuleInterface, requester::RequesterInterface}, 6 | types::config::requester::RequesterConfig, 7 | }; 8 | 9 | #[tokio::test] 10 | async fn configure_test() { 11 | let manager = CacheManager::default(); 12 | 13 | let old_config = RequesterConfig::default(); 14 | let new_config = RequesterConfig { 15 | timeout: Duration::from_secs(120), 16 | ..Default::default() 17 | }; 18 | 19 | for module in manager.modules().await.iter() { 20 | if let Some(requester) = module.lock().await.requester().await { 21 | assert_eq!(requester.lock().await.config().await, &old_config); 22 | } 23 | } 24 | 25 | manager.configure(new_config.clone()).await; 26 | 27 | for module in manager.modules().await.iter() { 28 | if let Some(requester) = module.lock().await.requester().await { 29 | assert_eq!(requester.lock().await.config().await, &new_config); 30 | } 31 | } 32 | } 33 | 34 | #[tokio::test] 35 | async fn module_test() { 36 | let manager = CacheManager::default(); 37 | 38 | assert!(manager.module("foo").await.is_none()); 39 | assert!(manager.module("google").await.is_some()); 40 | } 41 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/hackertarget_test.rs: -------------------------------------------------------------------------------- 1 | use subscan::{ 2 | enums::content::Content, 3 | modules::integrations::hackertarget::{HackerTarget, HACKERTARGET_URL}, 4 | types::result::status::SubscanModuleStatus, 5 | }; 6 | 7 | use crate::common::{ 8 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 9 | mock::funcs, 10 | utils, 11 | }; 12 | 13 | #[tokio::test] 14 | #[stubr::mock("module/integrations/hackertarget.json")] 15 | async fn run_test() { 16 | let mut hackertarget = HackerTarget::dispatcher(); 17 | 18 | funcs::wrap_module_url(&mut hackertarget, &stubr.path("/hackertarget")); 19 | 20 | let (results, status) = utils::run_module(hackertarget, TEST_DOMAIN).await; 21 | 22 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 23 | assert_eq!(status, SubscanModuleStatus::Finished); 24 | } 25 | 26 | #[tokio::test] 27 | async fn get_query_url_test() { 28 | let url = HackerTarget::get_query_url(TEST_DOMAIN); 29 | let expected = format!("{HACKERTARGET_URL}/?q={TEST_DOMAIN}"); 30 | 31 | assert_eq!(url, expected); 32 | } 33 | 34 | #[tokio::test] 35 | async fn get_next_url_test() { 36 | let url = TEST_URL.parse().unwrap(); 37 | let next = HackerTarget::get_next_url(url, Content::Empty); 38 | 39 | assert!(next.is_none()); 40 | } 41 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/shodan.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "queryParameters": { 5 | "key": { 6 | "equalTo": "shodan-api-key" 7 | }, 8 | "page": { 9 | "absent": true 10 | } 11 | }, 12 | "urlPath": "/shodan" 13 | }, 14 | "response": { 15 | "headers": { 16 | "content-type": "application/json" 17 | }, 18 | "jsonBody": { 19 | "data": [ 20 | { 21 | "last_seen": "2024-10-06T03:23:59.910000", 22 | "ports": [ 23 | 80, 24 | 443, 25 | 2082 26 | ], 27 | "subdomain": "", 28 | "tags": [ 29 | "cdn" 30 | ], 31 | "type": "A", 32 | "value": "104.18.174.21" 33 | } 34 | ], 35 | "domain": "foo.com", 36 | "more": true, 37 | "subdomains": [ 38 | "bar" 39 | ], 40 | "tags": [ 41 | "dmarc" 42 | ] 43 | }, 44 | "status": 200 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/types/core.rs: -------------------------------------------------------------------------------- 1 | use std::{result, sync::Arc}; 2 | 3 | use derive_more::From; 4 | use flume::{Receiver, Sender}; 5 | use tokio::sync::Mutex; 6 | 7 | use crate::{ 8 | enums::dispatchers::{ 9 | RequesterDispatcher, SubdomainExtractorDispatcher, SubscanModuleDispatcher, 10 | }, 11 | error::SubscanError, 12 | }; 13 | 14 | /// Result type 15 | pub type Result = result::Result; 16 | /// Core subdomain data type 17 | pub type Subdomain = String; 18 | /// `SubscanModule` type wrapper 19 | pub type SubscanModule = Arc>; 20 | 21 | impl From for SubscanModule { 22 | fn from(module: SubscanModuleDispatcher) -> Self { 23 | Self::new(Mutex::new(module)) 24 | } 25 | } 26 | 27 | /// Flume unbounded channel with generic typed 28 | #[derive(From)] 29 | #[from((Sender, Receiver))] 30 | pub struct UnboundedFlumeChannel { 31 | pub tx: Sender, 32 | pub rx: Receiver, 33 | } 34 | 35 | /// Container for core components of `Subscan` modules 36 | pub struct SubscanModuleCoreComponents { 37 | /// Requester object instance for HTTP requests 38 | pub requester: Mutex, 39 | /// Any extractor object to extract subdomain from content 40 | pub extractor: SubdomainExtractorDispatcher, 41 | } 42 | -------------------------------------------------------------------------------- /book/src/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | 4 | 5 | - [Introduction](README.md) 6 | - [User Guide](user-guide/README.md) 7 | - [Quickstart](user-guide/quickstart/README.md) 8 | - [Install](user-guide/quickstart/install.md) 9 | - [Usage](user-guide/quickstart/usage/README.md) 10 | - [CLI](user-guide/quickstart/usage/cli.md) 11 | - [Docker](user-guide/quickstart/usage/docker.md) 12 | - [Crate](user-guide/quickstart/usage/crate.md) 13 | - [Commands](user-guide/commands/README.md) 14 | - [scan](user-guide/commands/scan.md) 15 | - [brute](user-guide/commands/brute.md) 16 | - [module](user-guide/commands/module.md) 17 | - [Environments](user-guide/environments.md) 18 | - [Development](development/README.md) 19 | - [Setup Environment](development/environment.md) 20 | - [Components](development/components/README.md) 21 | - [Requesters](development/components/requesters.md) 22 | - [Extractors](development/components/extractors.md) 23 | - [Subscan Module](development/components/module.md) 24 | - [Generic Modules](development/generics/README.md) 25 | - [Integration](development/generics/integration.md) 26 | - [Search Engine](development/generics/engine.md) 27 | - [Integrate Your Module Step by Step](development/integration.md) 28 | 29 | 30 | -------------------------------------------------------------------------------- /tests/stubs/module/integrations/commoncrawl/commoncrawl-index-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "request": { 3 | "method": "GET", 4 | "urlPath": "/commoncrawl/index" 5 | }, 6 | "response": { 7 | "headers": { 8 | "content-type": "application/json" 9 | }, 10 | "jsonBody": [ 11 | { 12 | "cdx-api": "http://127.0.0.1:{{port}}/{{request.pathSegments.[0]}}/cdx-1", 13 | "id": "{{now format='yyyy/MM/dd'}}" 14 | }, 15 | { 16 | "cdx-api": "http://127.0.0.1:{{port}}/{{request.pathSegments.[0]}}/cdx-2", 17 | "id": "{{now format='yyyy/MM/dd'}}" 18 | }, 19 | { 20 | "cdx-api": "http://127.0.0.1:{{port}}/{{request.pathSegments.[0]}}/cdx-3", 21 | "id": "{{now offset='-3 years'}}" 22 | }, 23 | { 24 | "cdx-api": "http://127.0.0.1:{{port}}/{{request.pathSegments.[0]}}/cdx-4", 25 | "id": "{{now format='yyyy/MM/dd'}}" 26 | }, 27 | { 28 | "no-id-field": "foo" 29 | }, 30 | { 31 | "no-cdx-api-field": "foo" 32 | } 33 | ], 34 | "status": 200, 35 | "transformers": [ 36 | "response-template" 37 | ] 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/utilities/cli.rs: -------------------------------------------------------------------------------- 1 | use prettytable::{format::consts::FORMAT_NO_LINESEP_WITH_TITLE, row, table, Table}; 2 | 3 | /// Creates table for [`SubscanModule`](crate::types::core::SubscanModule) 4 | /// 5 | /// # Examples 6 | /// 7 | /// ``` 8 | /// use subscan::utilities::cli; 9 | /// 10 | /// #[tokio::main] 11 | /// async fn main() { 12 | /// let table = cli::create_module_table().await; 13 | /// 14 | /// assert!(table.is_empty()); 15 | /// } 16 | /// ``` 17 | pub async fn create_module_table() -> Table { 18 | let mut table = table!(); 19 | 20 | let titles = row!["Name", "Requester", "Extractor", "Is Generic?"]; 21 | 22 | table.set_format(*FORMAT_NO_LINESEP_WITH_TITLE); 23 | table.set_titles(titles); 24 | 25 | table 26 | } 27 | 28 | /// Creates table for [`SubscanResultItem`](crate::types::result::item::SubscanResultItem) 29 | /// 30 | /// # Examples 31 | /// 32 | /// ``` 33 | /// use subscan::utilities::cli; 34 | /// 35 | /// #[tokio::main] 36 | /// async fn main() { 37 | /// let table = cli::create_scan_result_item_table().await; 38 | /// 39 | /// assert!(table.is_empty()); 40 | /// } 41 | /// ``` 42 | pub async fn create_scan_result_item_table() -> Table { 43 | let mut table = table!(); 44 | 45 | let titles = row!["Subdomain", "IP"]; 46 | 47 | table.set_format(*FORMAT_NO_LINESEP_WITH_TITLE); 48 | table.set_titles(titles); 49 | 50 | table 51 | } 52 | -------------------------------------------------------------------------------- /src/types/func.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeSet, net::IpAddr, pin::Pin}; 2 | 3 | use futures::Future; 4 | use reqwest::Url; 5 | use serde_json::Value; 6 | 7 | use super::core::{Result, Subdomain}; 8 | use crate::enums::content::Content; 9 | 10 | /// Inner extract method type definition for [`JSONExtractor`](crate::extractors::json::JSONExtractor) 11 | /// In summary it takes a [`Value`] as a parameter and parse subdomains 12 | pub type InnerExtractFunc = Box Result> + Sync + Send>; 13 | /// Get query url function, [`GenericIntegrationModule`](crate::modules::generics::integration::GenericIntegrationModule) 14 | /// uses this type to get start query URL 15 | pub type GetQueryUrlFunc = Box String + Sync + Send>; 16 | /// Get next url function, [`GenericIntegrationModule`](crate::modules::generics::integration::GenericIntegrationModule) 17 | /// uses this function to get next query URL for fetch API fully 18 | pub type GetNextUrlFunc = Box Option + Sync + Send>; 19 | /// IP address resolver function type 20 | pub type AsyncIPResolveFunc = 21 | Box Pin> + Send>> + Send + Sync>; 22 | 23 | /// Container for generic integration module functions 24 | pub struct GenericIntegrationCoreFuncs { 25 | pub url: GetQueryUrlFunc, 26 | pub next: GetNextUrlFunc, 27 | } 28 | -------------------------------------------------------------------------------- /src/bin/subscan.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use clap::Parser; 4 | use subscan::{ 5 | cli::{ 6 | commands::{module::ModuleSubCommands, Commands}, 7 | Cli, 8 | }, 9 | Subscan, 10 | }; 11 | 12 | #[tokio::main] 13 | async fn main() { 14 | let cli = Cli::parse(); 15 | let subscan = Subscan::from(cli.clone()); 16 | let out = &mut io::stdout(); 17 | 18 | cli.init().await; 19 | cli.banner().await; 20 | 21 | match cli.command { 22 | Commands::Module(module) => match module.command { 23 | ModuleSubCommands::List(list) => { 24 | list.as_table(subscan.modules().await, out).await; 25 | } 26 | ModuleSubCommands::Get(get) => { 27 | get.as_table(subscan.module(&get.name).await, out).await; 28 | } 29 | ModuleSubCommands::Run(args) => { 30 | subscan.run(&args.name, &args.domain).await.save(&args.output).await; 31 | } 32 | }, 33 | Commands::Scan(args) => { 34 | subscan.scan(&args.domain).await.save(&args.output).await; 35 | } 36 | Commands::Brute(args) => { 37 | if args.stream_to_txt.is_some() { 38 | subscan.brute(&args.domain).await; 39 | } else { 40 | subscan.brute(&args.domain).await.save(&args.output).await; 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/enums/result.rs: -------------------------------------------------------------------------------- 1 | use derive_more::Deref; 2 | 3 | use crate::types::{ 4 | core::Subdomain, 5 | result::{ 6 | item::{SubscanModuleResultItem, SubscanModuleStatusItem}, 7 | status::SubscanModuleStatus, 8 | }, 9 | }; 10 | 11 | /// Subscan module result variants 12 | #[derive(Clone, Debug, PartialEq)] 13 | pub enum SubscanModuleResult { 14 | SubscanModuleResultItem(SubscanModuleResultItem), 15 | SubscanModuleStatusItem(SubscanModuleStatusItem), 16 | } 17 | 18 | /// Optional subscan module result type 19 | #[derive(Clone, Debug, Deref)] 20 | pub struct OptionalSubscanModuleResult(pub Option); 21 | 22 | impl From<(&str, &Subdomain)> for OptionalSubscanModuleResult { 23 | fn from(values: (&str, &Subdomain)) -> Self { 24 | Self(Some(SubscanModuleResult::SubscanModuleResultItem( 25 | values.into(), 26 | ))) 27 | } 28 | } 29 | 30 | impl From<(&str, SubscanModuleStatus)> for OptionalSubscanModuleResult { 31 | fn from(values: (&str, SubscanModuleStatus)) -> Self { 32 | Self(Some(SubscanModuleResult::SubscanModuleStatusItem( 33 | values.into(), 34 | ))) 35 | } 36 | } 37 | 38 | impl From<(&str, &str)> for OptionalSubscanModuleResult { 39 | fn from(values: (&str, &str)) -> Self { 40 | Self(Some(SubscanModuleResult::SubscanModuleStatusItem( 41 | values.into(), 42 | ))) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: Security 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | env: 9 | RUSTFLAGS: "-Dwarnings" 10 | CARGO_TERM_COLOR: "always" 11 | 12 | jobs: 13 | rust-cargo-deny: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v6 17 | - uses: EmbarkStudios/cargo-deny-action@v2 18 | with: 19 | command: check 20 | log-level: error 21 | arguments: --all-features 22 | 23 | rust-cargo-udeps: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v6 27 | - uses: dtolnay/rust-toolchain@nightly 28 | - uses: Swatinem/rust-cache@v2 29 | - uses: aig787/cargo-udeps-action@v1 30 | with: 31 | version: "latest" 32 | args: "--all-targets" 33 | env: 34 | RUSTFLAGS: "" 35 | 36 | rust-cargo-machete: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v6 40 | - uses: Swatinem/rust-cache@v2 41 | - uses: bnjbvr/cargo-machete@v0.9.1 42 | 43 | gitleaks-scan: 44 | permissions: 45 | contents: read 46 | discussions: write 47 | pull-requests: write 48 | runs-on: ubuntu-latest 49 | steps: 50 | - uses: actions/checkout@v6 51 | with: 52 | fetch-depth: 0 53 | - uses: gitleaks/gitleaks-action@v2 54 | env: 55 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | -------------------------------------------------------------------------------- /tests/components/common/mock/resolver.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::{IpAddr, Ipv4Addr}, 3 | str::FromStr, 4 | }; 5 | 6 | use async_trait::async_trait; 7 | use subscan::{ 8 | interfaces::lookup::LookUpHostFuture, 9 | types::{config::resolver::ResolverConfig, func::AsyncIPResolveFunc}, 10 | }; 11 | 12 | use crate::common::constants::LOCAL_HOST; 13 | 14 | #[derive(Default)] 15 | pub struct MockResolver { 16 | config: ResolverConfig, 17 | } 18 | 19 | impl MockResolver { 20 | pub fn new(config: ResolverConfig) -> Self { 21 | Self { config } 22 | } 23 | 24 | pub fn boxed(config: ResolverConfig) -> Box { 25 | Box::new(Self::new(config)) 26 | } 27 | 28 | pub fn default_boxed() -> Box { 29 | Box::new(Self::default()) 30 | } 31 | } 32 | 33 | impl From for MockResolver { 34 | fn from(config: ResolverConfig) -> Self { 35 | Self { config } 36 | } 37 | } 38 | 39 | #[async_trait] 40 | impl LookUpHostFuture for MockResolver { 41 | async fn lookup_host_future(&self) -> AsyncIPResolveFunc { 42 | if self.config.disabled { 43 | Box::new(|_: String| Box::pin(async move { None })) 44 | } else { 45 | Box::new(move |_: String| { 46 | Box::pin(async move { Some(IpAddr::V4(Ipv4Addr::from_str(LOCAL_HOST).unwrap())) }) 47 | }) 48 | } 49 | } 50 | 51 | async fn config(&self) -> ResolverConfig { 52 | self.config.clone() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /tests/components/extractors/json_test.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, error::ModuleErrorKind::JSONExtract, extractors::json::JSONExtractor, 6 | interfaces::extractor::SubdomainExtractorInterface, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_BAZ_SUBDOMAIN, TEST_DOMAIN}, 11 | utils::read_testdata, 12 | }; 13 | 14 | #[tokio::test] 15 | async fn extract_test() { 16 | let json = read_testdata("json/subdomains.json"); 17 | 18 | let inner_parser = |json: Value, _domain: &str| { 19 | if let Some(subdomains) = json["data"]["subdomains"].as_array() { 20 | let filter = |json: &Value| Some(json["subdomain"].as_str().unwrap().to_string()); 21 | 22 | return Ok(BTreeSet::from_iter(subdomains.iter().filter_map(filter))); 23 | } 24 | 25 | Err(JSONExtract.into()) 26 | }; 27 | 28 | let extractor = JSONExtractor::new(Box::new(inner_parser)); 29 | 30 | let result = extractor.extract(json, TEST_DOMAIN).await; 31 | let no_result = extractor.extract(Content::default(), TEST_DOMAIN).await; 32 | 33 | let expected = BTreeSet::from([ 34 | TEST_BAR_SUBDOMAIN.to_string(), 35 | TEST_BAZ_SUBDOMAIN.to_string(), 36 | ]); 37 | 38 | assert!(result.is_ok()); 39 | assert!(no_result.is_err()); 40 | 41 | assert_eq!(no_result.err().unwrap(), JSONExtract.into()); 42 | assert_eq!(result.unwrap(), expected); 43 | } 44 | -------------------------------------------------------------------------------- /book/src/README.md: -------------------------------------------------------------------------------- 1 | 2 |
3 |


4 | Subscan Logo 5 |


6 |
7 | 8 | 9 | Subscan is a powerful subdomain enumeration tool built with [Rust](https://www.rust-lang.org/), specifically designed for penetration testing purposes. It combines various discovery techniques into a single, lightweight binary, making subdomain hunting easier and faster for security researchers 10 | 11 | ### Features 12 | 13 | - 🕵️ Smart Discovery Tricks 14 | - Use multiple search engines (`Google`, `Yahoo`, `Bing`, `DuckDuckGo`, etc.) 15 | - Integrate with APIs like `Shodan`, `Censys`, `VirusTotal` and more 16 | - Perform zone transfer checks 17 | - Subdomain brute-forcing with optimized wordlists 18 | - 🔍 Resolve IP addresses for all subdomains 19 | - 📎 Export reports in `CSV`, `HTML`, `JSON`, or `TXT` formats 20 | - 🛠️ Configurable 21 | - Customize HTTP requests (user-agent, timeout, etc.) 22 | - Rotate requests via proxies (`--proxy` argument) 23 | - Fine-tune IP resolver with `--resolver` arguments 24 | - Filter and run specific modules with `--skips` and `--modules` 25 | - 🐳 Docker Friendly 26 | - Native support for `amd64` and `arm64` Linux platforms 27 | - A tiny container that won't eat up your storage — under 1GB and ready to roll 🚀 28 | - 💻 Compatible with multiple platforms and easy to install as a single binary 29 | -------------------------------------------------------------------------------- /src/cli/commands/brute.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use clap::Args; 4 | 5 | use crate::{ 6 | constants::{DEFAULT_RESOLVER_CONCURRENCY, DEFAULT_RESOLVER_TIMEOUT}, 7 | enums::output::OutputFormat, 8 | }; 9 | 10 | /// Brute force attack command arguments 11 | #[derive(Args, Clone, Debug, Default)] 12 | pub struct BruteCommandArgs { 13 | /// Target domain address for brute force attack 14 | #[arg(short, long)] 15 | pub domain: String, 16 | /// Wordlist file to be used during attack 17 | #[arg(short, long)] 18 | pub wordlist: PathBuf, 19 | /// If sets, output will be logged on stdout 20 | #[arg(long, default_value_t = false)] 21 | pub print: bool, 22 | /// Optional txt file to create file stream for the subdomains that found. 23 | /// If sets the `--output` parameter will be disabled 24 | #[arg(short, long, default_value = None)] 25 | pub stream_to_txt: Option, 26 | /// Set output format 27 | #[arg(value_enum, short, long, default_value_t = OutputFormat::JSON)] 28 | pub output: OutputFormat, 29 | /// IP resolver timeout value as a milliseconds 30 | #[arg(long, default_value_t = DEFAULT_RESOLVER_TIMEOUT.as_millis() as u64)] 31 | pub resolver_timeout: u64, 32 | /// IP resolver concurrency level, thread counts of resolver instances 33 | #[arg(long, default_value_t = DEFAULT_RESOLVER_CONCURRENCY)] 34 | pub resolver_concurrency: u64, 35 | /// A text file containing list of resolvers. See `resolverlist.template` 36 | #[arg(long, default_value = None)] 37 | pub resolver_list: Option, 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Coverage 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | env: 12 | RUSTFLAGS: "-Dwarnings" 13 | CARGO_TERM_COLOR: "always" 14 | 15 | jobs: 16 | cargo-llvm-cov: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v6 20 | - uses: dtolnay/rust-toolchain@nightly 21 | - uses: Swatinem/rust-cache@v2 22 | - uses: taiki-e/install-action@cargo-llvm-cov 23 | 24 | # We are not using nextest in here because of it has some 25 | # shortcomings while working with cargo-llvm-cov and can cause 26 | # false positive results, just use only cargo-llvm-cov 27 | # 28 | # We are using `coverage-ci` profile to reduce binary sizes 29 | # cause of github action runner disk space limitations. This is 30 | # a bit slow but we have to do it because we don't have 31 | # self-hosted runner right now 32 | - name: Generate code coverage 33 | run: | 34 | cargo +nightly llvm-cov \ 35 | --profile coverage-ci \ 36 | --all-features \ 37 | --workspace \ 38 | --doctests \ 39 | --lcov \ 40 | --output-path lcov.info 41 | 42 | - name: Upload coverage report to Codecov 43 | uses: codecov/codecov-action@v5 44 | if: ${{ !(startsWith(github.head_ref, 'release-plz') || startsWith(github.head_ref, 'dependabot')) }} 45 | with: 46 | token: ${{ secrets.CODECOV_TOKEN }} 47 | files: lcov.info 48 | fail_ci_if_error: true 49 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/waybackarchive_test.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeSet, time::Duration}; 2 | 3 | use subscan::{ 4 | error::ModuleErrorKind::GetContent, modules::integrations::waybackarchive::WaybackArchive, 5 | types::result::status::SubscanModuleStatus, 6 | }; 7 | 8 | use crate::common::{ 9 | constants::{TEST_BAR_SUBDOMAIN, TEST_BAZ_SUBDOMAIN, TEST_DOMAIN}, 10 | mock::funcs, 11 | utils, 12 | }; 13 | 14 | #[tokio::test] 15 | #[stubr::mock("module/integrations/waybackarchive/waybackarchive.json")] 16 | async fn run_success_test() { 17 | let mut waybackarchive = WaybackArchive::dispatcher(); 18 | 19 | funcs::wrap_module_url(&mut waybackarchive, &stubr.path("/waybackarchive")); 20 | 21 | let (results, status) = utils::run_module(waybackarchive, TEST_DOMAIN).await; 22 | 23 | let expected = BTreeSet::from([ 24 | TEST_BAR_SUBDOMAIN.to_string(), 25 | TEST_BAZ_SUBDOMAIN.to_string(), 26 | ]); 27 | 28 | assert_eq!(results, expected); 29 | assert_eq!(status, SubscanModuleStatus::Finished); 30 | } 31 | 32 | #[tokio::test] 33 | #[stubr::mock("module/integrations/waybackarchive/waybackarchive-delayed.json")] 34 | async fn run_timeout_test() { 35 | let mut waybackarchive = WaybackArchive::dispatcher(); 36 | 37 | funcs::wrap_module_url(&mut waybackarchive, &stubr.path("/waybackarchive-delayed")); 38 | funcs::set_requester_timeout(&mut waybackarchive, Duration::from_millis(500)).await; 39 | 40 | let (results, status) = utils::run_module(waybackarchive, TEST_DOMAIN).await; 41 | 42 | assert_eq!(results, BTreeSet::new()); 43 | assert_eq!(status, GetContent.into()); 44 | } 45 | -------------------------------------------------------------------------------- /book/src/user-guide/quickstart/usage/crate.md: -------------------------------------------------------------------------------- 1 | # Crate Usage 2 | 3 | You can easily add `Subscan` to your code and use its results in your application. Since `Subscan` works asynchronously, you need to use it in `async` code blocks. We recommend using [Tokio](https://tokio.rs/) as the async runtime 4 | 5 | This chapter provides step-by-step guidance on how to integrate `Subscan` into your code. For more detailed usage and additional code examples, visit the project's [docs.rs](https://docs.rs/subscan/latest/subscan/) page or check the [examples/](https://github.com/eredotpkfr/subscan/tree/main/examples) folder in the repository 6 | 7 | 1. Add `subscan` crate into your project dependencies 8 | 9 | ```bash 10 | ~$ cargo add subscan 11 | ``` 12 | 13 | 2. Create a new instance and start to use it 14 | 15 | ```rust,ignore 16 | #[tokio::main] 17 | async fn main() { 18 | // set module conccurrency to 1 19 | // set HTTP timeout to 120 20 | let config = SubscanConfig { 21 | concurrency: 1, 22 | filter: CacheFilter::FilterByName(ModuleNameFilter { 23 | modules: vec!["alienvault".into()], 24 | skips: vec![], 25 | }), 26 | requester: RequesterConfig { 27 | timeout: Duration::from_secs(120), 28 | ..Default::default() 29 | }, 30 | ..Default::default() 31 | }; 32 | 33 | let subscan = Subscan::from(config); 34 | let result = subscan.scan("domain.com").await; 35 | 36 | for item in result.items { 37 | // do something with item 38 | } 39 | } 40 | ``` 41 | -------------------------------------------------------------------------------- /book/src/user-guide/quickstart/usage/docker.md: -------------------------------------------------------------------------------- 1 | # Docker Usage 2 | 3 | Once you’ve [pulled](../../quickstart/install.md#pull-docker-image) the pre-built image from [Docker Hub](https://hub.docker.com/), you can easily run the container to perform subdomain enumeration 4 | 5 | ```bash 6 | ~$ docker run -it --rm eredotpkfr/subscan scan -d example.com 7 | ``` 8 | 9 | Specify environment variable via docker `--env` 10 | 11 | ```bash 12 | ~$ docker run -it --rm \ 13 | --env SUBSCAN_VIRUSTOTAL_APIKEY=foo \ 14 | eredotpkfr/subscan scan -d example.com --modules=virustotal 15 | ``` 16 | 17 | Specify `.env` file from your host machine, use `/data` folder 18 | 19 | ```bash 20 | ~$ docker run -it --rm \ 21 | --volume="$PWD/.env:/data/.env" \ 22 | eredotpkfr/subscan scan -d example.com --skips=commoncrawl 23 | ``` 24 | 25 | Saving output reports to host machine, use `/data` folder 26 | 27 | ```bash 28 | ~$ docker run -it --rm \ 29 | --volume="$PWD/data:/data" \ 30 | eredotpkfr/subscan scan -d example.com 31 | ``` 32 | 33 | To specify wordlist into docker container, use `/data` folder 34 | 35 | ```bash 36 | ~$ docker run -it --rm \ 37 | --volume="$PWD/wordlist.txt:/data/wordlist.txt" \ 38 | eredotpkfr/subscan brute -d example.com \ 39 | -w wordlist.txt --print 40 | ``` 41 | 42 | ## Build a Docker Image 43 | 44 | To build a Docker image locally, run the following command 45 | 46 | ```bash 47 | ~$ docker build -t subscan . 48 | ``` 49 | 50 | > If you encounter memory issues while building on an Apple Silicon machine, you can run Colima with the following parameters 51 | > 52 | > ```bash 53 | > ~$ colima start --cpu 11 --memory 16 54 | > ``` 55 | -------------------------------------------------------------------------------- /tests/components/cli/commands/module/list_test.rs: -------------------------------------------------------------------------------- 1 | use std::io::Cursor; 2 | 3 | use clap::Parser; 4 | use subscan::{cli::Cli, modules::engines::google::Google, types::core::SubscanModule}; 5 | 6 | use crate::common::utils; 7 | 8 | #[tokio::test] 9 | #[should_panic] 10 | async fn module_list_parse_error_test() { 11 | Cli::try_parse_from(vec!["subscan", "module", "list", "-x"]).unwrap(); 12 | } 13 | 14 | #[tokio::test] 15 | async fn module_list_test() { 16 | let args = vec!["subscan", "module", "list"]; 17 | let cli = Cli::try_parse_from(args).unwrap(); 18 | 19 | let expected = "\ 20 | +--------+------------+---------------+-------------+\n\ 21 | | Name | Requester | Extractor | Is Generic? |\n\ 22 | +--------+------------+---------------+-------------+\n\ 23 | | google | HTTPClient | HTMLExtractor | true |\n\ 24 | +--------+------------+---------------+-------------+\n\ 25 | "; 26 | 27 | match cli.command { 28 | subscan::cli::commands::Commands::Module(sub) => match sub.command { 29 | subscan::cli::commands::module::ModuleSubCommands::List(args) => { 30 | let mut out = Cursor::new(vec![]); 31 | let module = SubscanModule::from(Google::dispatcher()); 32 | 33 | args.as_table(&vec![module], &mut out).await; 34 | 35 | let result = String::from_utf8(out.into_inner()).unwrap(); 36 | 37 | assert_eq!(utils::fix_new_lines(&result), expected); 38 | } 39 | _ => panic!("Expected ModuleSubCommands::List"), 40 | }, 41 | _ => panic!("Expected Commands::Module"), 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tests/components/cli/commands/module/get_test.rs: -------------------------------------------------------------------------------- 1 | use std::io::Cursor; 2 | 3 | use clap::Parser; 4 | use subscan::{cli::Cli, modules::engines::google::Google}; 5 | use tokio::sync::Mutex; 6 | 7 | use crate::common::utils; 8 | 9 | #[tokio::test] 10 | #[should_panic] 11 | async fn module_get_parse_error_test() { 12 | Cli::try_parse_from(vec!["subscan", "module", "get", "-x"]).unwrap(); 13 | } 14 | 15 | #[tokio::test] 16 | async fn module_get_test() { 17 | let args = vec!["subscan", "module", "get", "google"]; 18 | let cli = Cli::try_parse_from(args).unwrap(); 19 | 20 | let expected = "\ 21 | +--------+------------+---------------+-------------+\n\ 22 | | Name | Requester | Extractor | Is Generic? |\n\ 23 | +--------+------------+---------------+-------------+\n\ 24 | | google | HTTPClient | HTMLExtractor | true |\n\ 25 | +--------+------------+---------------+-------------+\n\ 26 | "; 27 | 28 | match cli.command { 29 | subscan::cli::commands::Commands::Module(sub) => match sub.command { 30 | subscan::cli::commands::module::ModuleSubCommands::Get(args) => { 31 | let mut out = Cursor::new(vec![]); 32 | let module = Mutex::new(Google::dispatcher()); 33 | 34 | args.as_table(&module, &mut out).await; 35 | 36 | let result = String::from_utf8(out.into_inner()).unwrap(); 37 | 38 | assert_eq!(args.name, "google"); 39 | assert_eq!(utils::fix_new_lines(&result), expected); 40 | } 41 | _ => panic!("Expected ModuleSubCommands::Get"), 42 | }, 43 | _ => panic!("Expected Commands::Module"), 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | # Configuration for https://github.com/EmbarkStudios/cargo-deny 2 | 3 | [licenses] 4 | confidence-threshold = 0.8 5 | allow = [ 6 | "Unicode-3.0", 7 | "MPL-2.0", 8 | "MIT", 9 | "Apache-2.0", 10 | "ISC", 11 | "BSD-3-Clause", 12 | "GPL-3.0-only", 13 | "GPL-3.0" 14 | ] 15 | 16 | # List of license exceptions 17 | exceptions = [ 18 | { allow = ["CDLA-Permissive-2.0"], crate = "webpki-roots" }, 19 | { allow = ["LGPL-3.0"], crate = "colog" }, 20 | { allow = ["Zlib"], crate = "nanorand" }, 21 | { allow = ["Zlib"], crate = "libz-rs-sys" }, 22 | { allow = ["Zlib"], crate = "zlib-rs" }, 23 | { allow = ["GPL-3.0"], crate = "auto_generate_cdp" }, 24 | { allow = ["CC0-1.0"], crate = "ppmd-rust" }, 25 | { allow = ["bzip2-1.0.6"], crate = "libbz2-rs-sys" } 26 | ] 27 | 28 | [[licenses.clarify]] 29 | crate = "ring" 30 | expression = "MIT AND ISC AND OpenSSL" 31 | 32 | license-files = [ 33 | # Each entry is a crate relative path, and the (opaque) hash of its contents 34 | { path = "LICENSE", hash = 0xbd0eed23 } 35 | ] 36 | 37 | [advisories] 38 | # Add the unmaintained crate here 39 | ignore = [ 40 | { id = "RUSTSEC-2024-0384", reason = "Not maintained, but no upgrade option" }, 41 | { id = "RUSTSEC-2025-0010", reason = "ring 0.16.20 was released over 4 years ago and isn't maintained" }, 42 | { id = "RUSTSEC-2025-0009", reason = "ring::aead::quic::HeaderProtectionKey::new_mask() may panic when overflow checking is enabled" }, 43 | { id = "RUSTSEC-2025-0014", reason = "Latest humantime crates.io release is four years old and GitHub repository has not seen commits in four years" }, 44 | { id = "RUSTSEC-2025-0052", reason = "unmaintained advisory detected" }, 45 | { id = "RUSTSEC-2025-0057", reason = "No safe upgrade is available!" } 46 | ] 47 | -------------------------------------------------------------------------------- /book/src/development/generics/engine.md: -------------------------------------------------------------------------------- 1 | # Generic Search Engine Module 2 | 3 | The [`GenericSearchEngineModule`](https://docs.rs/subscan/latest/subscan/modules/generics/engine/struct.GenericSearchEngineModule.html) is primarily used for search engine integrations. It performs subdomain discovery by conducting dork searches on search engines and provides a generic implementation for search engines that use the same dork structure. To understand how it works, review the source code on the [docs.rs](https://docs.rs/subscan/latest/subscan/modules/generics/engine/struct.GenericSearchEngineModule.html) page. Additionally, the source code of other module implementations that use this implementation can help guide you in its usage 4 | 5 | A search engine module that uses this internally would look like the example below 6 | 7 | ```rust,ignore 8 | pub const EXAMPLE_MODULE_NAME: &str = "example"; 9 | pub const EXAMPLE_SEARCH_URL: &str = "https://www.example.com/search"; 10 | pub const EXAMPLE_SEARCH_PARAM: &str = "q"; 11 | pub const EXAMPLE_CITE_TAG: &str = "cite"; 12 | 13 | pub struct ExampleModule {} 14 | 15 | impl ExampleModule { 16 | pub fn dispatcher() -> SubscanModuleDispatcher { 17 | let url = Url::parse(EXAMPLE_SEARCH_URL); 18 | 19 | let extractor: HTMLExtractor = HTMLExtractor::new(EXAMPLE_CITE_TAG.into(), vec![]); 20 | let requester: RequesterDispatcher = HTTPClient::default().into(); 21 | 22 | let generic = GenericSearchEngineModule { 23 | name: EXAMPLE_MODULE_NAME.into(), 24 | param: EXAMPLE_SEARCH_PARAM.into(), 25 | url: url.unwrap(), 26 | components: SubscanModuleCoreComponents { 27 | requester: requester.into(), 28 | extractor: extractor.into(), 29 | }, 30 | }; 31 | 32 | generic.into() 33 | } 34 | } 35 | ``` 36 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/crtsh_test.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use subscan::{ 3 | enums::content::Content, 4 | error::ModuleErrorKind::JSONExtract, 5 | modules::integrations::crtsh::{Crtsh, CRTSH_URL}, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 11 | mock::funcs, 12 | utils, 13 | }; 14 | 15 | #[tokio::test] 16 | #[stubr::mock("module/integrations/crtsh.json")] 17 | async fn run_test() { 18 | let mut crtsh = Crtsh::dispatcher(); 19 | 20 | funcs::wrap_module_url(&mut crtsh, &stubr.path("/crtsh")); 21 | 22 | let (results, status) = utils::run_module(crtsh, TEST_DOMAIN).await; 23 | 24 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 25 | assert_eq!(status, SubscanModuleStatus::Finished); 26 | } 27 | 28 | #[tokio::test] 29 | async fn get_query_url_test() { 30 | let url = Crtsh::get_query_url(TEST_DOMAIN); 31 | let expected = format!("{CRTSH_URL}/?q={TEST_DOMAIN}&output=json"); 32 | 33 | assert_eq!(url, expected); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_next_url_test() { 38 | let url = TEST_URL.parse().unwrap(); 39 | let next = Crtsh::get_next_url(url, Content::Empty); 40 | 41 | assert!(next.is_none()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn extract_test() { 46 | let stub = "module/integrations/crtsh.json"; 47 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 48 | 49 | let extracted = Crtsh::extract(json, TEST_DOMAIN); 50 | let not_extracted = Crtsh::extract(Value::Null, TEST_DOMAIN); 51 | 52 | assert!(extracted.is_ok()); 53 | assert!(not_extracted.is_err()); 54 | 55 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 56 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 57 | } 58 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/anubis_test.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use subscan::{ 3 | enums::content::Content, 4 | error::ModuleErrorKind::JSONExtract, 5 | modules::integrations::anubis::{Anubis, ANUBIS_URL}, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 11 | mock::funcs, 12 | utils, 13 | }; 14 | 15 | #[tokio::test] 16 | #[stubr::mock("module/integrations/anubis.json")] 17 | async fn run_test() { 18 | let mut anubis = Anubis::dispatcher(); 19 | 20 | funcs::wrap_module_url(&mut anubis, &stubr.path("/anubis")); 21 | 22 | let (results, status) = utils::run_module(anubis, TEST_DOMAIN).await; 23 | 24 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 25 | assert_eq!(status, SubscanModuleStatus::Finished); 26 | } 27 | 28 | #[tokio::test] 29 | async fn get_query_url_test() { 30 | let url = Anubis::get_query_url(TEST_DOMAIN); 31 | let expected = format!("{ANUBIS_URL}/{TEST_DOMAIN}"); 32 | 33 | assert_eq!(url, expected); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_next_url_test() { 38 | let url = TEST_URL.parse().unwrap(); 39 | let next = Anubis::get_next_url(url, Content::Empty); 40 | 41 | assert!(next.is_none()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn extract_test() { 46 | let stub = "module/integrations/anubis.json"; 47 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 48 | 49 | let extracted = Anubis::extract(json, TEST_DOMAIN); 50 | let not_extracted = Anubis::extract(Value::Null, TEST_DOMAIN); 51 | 52 | assert!(extracted.is_ok()); 53 | assert!(not_extracted.is_err()); 54 | 55 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 56 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 57 | } 58 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/leakix_test.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use subscan::{ 3 | enums::content::Content, 4 | error::ModuleErrorKind::JSONExtract, 5 | modules::integrations::leakix::{Leakix, LEAKIX_URL}, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 11 | mock::funcs, 12 | utils, 13 | }; 14 | 15 | #[tokio::test] 16 | #[stubr::mock("module/integrations/leakix.json")] 17 | async fn run_test() { 18 | let mut leakix = Leakix::dispatcher(); 19 | 20 | funcs::wrap_module_url(&mut leakix, &stubr.path("/leakix")); 21 | 22 | let (results, status) = utils::run_module(leakix, TEST_DOMAIN).await; 23 | 24 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 25 | assert_eq!(status, SubscanModuleStatus::Finished); 26 | } 27 | 28 | #[tokio::test] 29 | async fn get_query_url_test() { 30 | let url = Leakix::get_query_url(TEST_DOMAIN); 31 | let expected = format!("{LEAKIX_URL}/subdomains/{TEST_DOMAIN}"); 32 | 33 | assert_eq!(url, expected); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_next_url_test() { 38 | let url = TEST_URL.parse().unwrap(); 39 | let next = Leakix::get_next_url(url, Content::Empty); 40 | 41 | assert!(next.is_none()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn extract_test() { 46 | let stub = "module/integrations/leakix.json"; 47 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 48 | 49 | let extracted = Leakix::extract(json, TEST_DOMAIN); 50 | let not_extracted = Leakix::extract(Value::Null, TEST_DOMAIN); 51 | 52 | assert!(extracted.is_ok()); 53 | assert!(not_extracted.is_err()); 54 | 55 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 56 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 57 | } 58 | -------------------------------------------------------------------------------- /src/enums/auth.rs: -------------------------------------------------------------------------------- 1 | use derive_more::From; 2 | 3 | use crate::types::env::Credentials; 4 | 5 | /// Authentication methods for API calls or HTTP requests. [`GenericIntegrationModule`](crate::modules::generics::integration::GenericIntegrationModule) 6 | /// uses them to apply correct auth method. See the method descriptions to learn how it works 7 | #[derive(From, PartialEq)] 8 | pub enum AuthenticationMethod { 9 | /// Some APIs uses request headers to get API key. If this auth type selected API key 10 | /// will add in request headers with a given header key 11 | APIKeyAsHeader(String), 12 | /// This auth type uses when API require API key as a query param. If this method chose 13 | /// API key will be added in URL as a query param with given parameter key 14 | APIKeyAsQueryParam(String), 15 | /// Use basic HTTP authentication method. If the credentials are not provided, module 16 | /// tries to fetch from environment variables using pre-formatted 17 | /// (see [`format_env`](crate::utilities::env::format_env)) variable names. Module specific 18 | /// variable names looks like `SUBSCAN_FOO_USERNAME`, `SUBSCAN_FOO_PASSWORD` 19 | #[from] 20 | BasicHTTPAuthentication(Credentials), 21 | /// This auth type does nothing for auth 22 | NoAuthentication, 23 | } 24 | 25 | impl AuthenticationMethod { 26 | /// Checks the any auth method selector or not 27 | /// 28 | /// # Examples 29 | /// 30 | /// ``` 31 | /// use subscan::enums::auth::AuthenticationMethod; 32 | /// 33 | /// let as_header = AuthenticationMethod::APIKeyAsHeader("X-API-Key".to_string()); 34 | /// let no_auth = AuthenticationMethod::NoAuthentication; 35 | /// 36 | /// assert!(as_header.is_set()); 37 | /// assert!(!no_auth.is_set()); 38 | /// ``` 39 | pub fn is_set(&self) -> bool { 40 | self != &Self::NoAuthentication 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/alienvault_test.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use subscan::{ 3 | enums::content::Content, 4 | error::ModuleErrorKind::JSONExtract, 5 | modules::integrations::alienvault::{AlienVault, ALIENVAULT_URL}, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 11 | mock::funcs, 12 | utils, 13 | }; 14 | 15 | #[tokio::test] 16 | #[stubr::mock("module/integrations/alienvault.json")] 17 | async fn run_test() { 18 | let mut alienvault = AlienVault::dispatcher(); 19 | 20 | funcs::wrap_module_url(&mut alienvault, &stubr.path("/alienvault")); 21 | 22 | let (results, status) = utils::run_module(alienvault, TEST_DOMAIN).await; 23 | 24 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 25 | assert_eq!(status, SubscanModuleStatus::Finished); 26 | } 27 | 28 | #[tokio::test] 29 | async fn get_query_url_test() { 30 | let url = AlienVault::get_query_url(TEST_DOMAIN); 31 | let expected = format!("{ALIENVAULT_URL}/{TEST_DOMAIN}/passive_dns"); 32 | 33 | assert_eq!(url, expected); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_next_url_test() { 38 | let url = TEST_URL.parse().unwrap(); 39 | let next = AlienVault::get_next_url(url, Content::Empty); 40 | 41 | assert!(next.is_none()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn extract_test() { 46 | let stub = "module/integrations/alienvault.json"; 47 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 48 | 49 | let extracted = AlienVault::extract(json, TEST_DOMAIN); 50 | let not_extracted = AlienVault::extract(Value::Null, TEST_DOMAIN); 51 | 52 | assert!(extracted.is_ok()); 53 | assert!(not_extracted.is_err()); 54 | 55 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 56 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 57 | } 58 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/threatcrowd_test.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use subscan::{ 3 | enums::content::Content, 4 | error::ModuleErrorKind::JSONExtract, 5 | modules::integrations::threatcrowd::{ThreatCrowd, THREATCROWD_URL}, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 11 | mock::funcs, 12 | utils, 13 | }; 14 | 15 | #[tokio::test] 16 | #[stubr::mock("module/integrations/threatcrowd.json")] 17 | async fn run_test() { 18 | let mut threatcrowd = ThreatCrowd::dispatcher(); 19 | 20 | funcs::wrap_module_url(&mut threatcrowd, &stubr.path("/threatcrowd")); 21 | 22 | let (results, status) = utils::run_module(threatcrowd, TEST_DOMAIN).await; 23 | 24 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 25 | assert_eq!(status, SubscanModuleStatus::Finished); 26 | } 27 | 28 | #[tokio::test] 29 | async fn get_query_url_test() { 30 | let url = ThreatCrowd::get_query_url(TEST_DOMAIN); 31 | let expected = format!("{THREATCROWD_URL}/?domain={TEST_DOMAIN}"); 32 | 33 | assert_eq!(url, expected); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_next_url_test() { 38 | let url = TEST_URL.parse().unwrap(); 39 | let next = ThreatCrowd::get_next_url(url, Content::Empty); 40 | 41 | assert!(next.is_none()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn extract_test() { 46 | let stub = "module/integrations/threatcrowd.json"; 47 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 48 | 49 | let extracted = ThreatCrowd::extract(json, TEST_DOMAIN); 50 | let not_extracted = ThreatCrowd::extract(Value::Null, TEST_DOMAIN); 51 | 52 | assert!(extracted.is_ok()); 53 | assert!(not_extracted.is_err()); 54 | 55 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 56 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 57 | } 58 | -------------------------------------------------------------------------------- /src/cli/commands/module/run.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use clap::Args; 4 | 5 | use crate::{ 6 | constants::{ 7 | DEFAULT_HTTP_TIMEOUT, DEFAULT_RESOLVER_CONCURRENCY, DEFAULT_RESOLVER_TIMEOUT, 8 | DEFAULT_USER_AGENT, 9 | }, 10 | enums::output::OutputFormat, 11 | }; 12 | 13 | /// Run command to start any module 14 | #[derive(Args, Clone, Debug, Default)] 15 | pub struct ModuleRunSubCommandArgs { 16 | /// Module name to be run 17 | pub name: String, 18 | /// Target domain address to be enumerated 19 | #[arg(short, long)] 20 | pub domain: String, 21 | /// Set output format 22 | #[arg(value_enum, short, long, default_value_t = OutputFormat::JSON)] 23 | pub output: OutputFormat, 24 | /// If sets, output will be logged on stdout 25 | #[arg(long, default_value_t = false)] 26 | pub print: bool, 27 | /// Set User-Agent header value for HTTP requests 28 | #[arg(short, long, default_value = DEFAULT_USER_AGENT)] 29 | pub user_agent: String, 30 | /// HTTP timeout value as a seconds 31 | #[arg(short = 't', long, default_value_t = DEFAULT_HTTP_TIMEOUT.as_secs())] 32 | pub http_timeout: u64, 33 | /// Set HTTP proxy 34 | #[arg(short, long, default_value = None)] 35 | pub proxy: Option, 36 | /// IP resolver timeout value as a milliseconds 37 | #[arg(long, default_value_t = DEFAULT_RESOLVER_TIMEOUT.as_millis() as u64)] 38 | pub resolver_timeout: u64, 39 | /// IP resolver concurrency level, thread counts of resolver instances 40 | #[arg(long, default_value_t = DEFAULT_RESOLVER_CONCURRENCY)] 41 | pub resolver_concurrency: u64, 42 | /// A text file containing list of resolvers. See `resolverlist.template` 43 | #[arg(long, default_value = None)] 44 | pub resolver_list: Option, 45 | /// Disable IP address resolve process 46 | #[arg(long = "disable-ip-resolve", default_value_t = false)] 47 | pub resolver_disabled: bool, 48 | } 49 | -------------------------------------------------------------------------------- /src/types/filters.rs: -------------------------------------------------------------------------------- 1 | use derive_more::From; 2 | 3 | /// This filter allows to filter modules by their names 4 | #[derive(Clone, Debug, From, PartialEq)] 5 | #[from((Vec, Vec))] 6 | pub struct ModuleNameFilter { 7 | /// Valid [`SubscanModule`](crate::types::core::SubscanModule) names list 8 | pub modules: Vec, 9 | /// Invalid [`SubscanModule`](crate::types::core::SubscanModule) names list 10 | pub skips: Vec, 11 | } 12 | 13 | impl ModuleNameFilter { 14 | /// Check module name is filtered or non-filtered by this filter 15 | /// 16 | /// # Examples 17 | /// 18 | /// ``` 19 | /// use subscan::types::filters::ModuleNameFilter; 20 | /// 21 | /// #[tokio::main] 22 | /// async fn main() { 23 | /// let filter: ModuleNameFilter = (vec![], vec![]).into(); 24 | /// assert!(!filter.is_filtered("foo").await); 25 | /// 26 | /// let filter: ModuleNameFilter = (vec![], vec!["foo".into()]).into(); 27 | /// assert!(filter.is_filtered("foo").await); 28 | /// 29 | /// let filter: ModuleNameFilter = (vec!["bar".into()], vec![]).into(); 30 | /// assert!(filter.is_filtered("foo").await); 31 | /// 32 | /// let filter: ModuleNameFilter = (vec!["foo".into()], vec!["foo".into()]).into(); 33 | /// assert!(filter.is_filtered("foo").await); 34 | /// } 35 | /// ``` 36 | pub async fn is_filtered(&self, name: &str) -> bool { 37 | if self.modules.is_empty() && self.skips.is_empty() { 38 | false 39 | } else if self.modules.is_empty() && !self.skips.is_empty() { 40 | self.skips.contains(&name.to_lowercase()) 41 | } else if !self.modules.is_empty() && self.skips.is_empty() { 42 | !self.modules.contains(&name.to_lowercase()) 43 | } else { 44 | !self.modules.contains(&name.to_lowercase()) 45 | || self.skips.contains(&name.to_lowercase()) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/subdomaincenter_test.rs: -------------------------------------------------------------------------------- 1 | use serde_json::Value; 2 | use subscan::{ 3 | enums::content::Content, 4 | error::ModuleErrorKind::JSONExtract, 5 | modules::integrations::subdomaincenter::{SubdomainCenter, SUBDOMAINCENTER_URL}, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | 9 | use crate::common::{ 10 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 11 | mock::funcs, 12 | utils, 13 | }; 14 | 15 | #[tokio::test] 16 | #[stubr::mock("module/integrations/subdomaincenter.json")] 17 | async fn run_test() { 18 | let mut subdomaincenter = SubdomainCenter::dispatcher(); 19 | 20 | funcs::wrap_module_url(&mut subdomaincenter, &stubr.path("/subdomaincenter")); 21 | 22 | let (results, status) = utils::run_module(subdomaincenter, TEST_DOMAIN).await; 23 | 24 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 25 | assert_eq!(status, SubscanModuleStatus::Finished); 26 | } 27 | 28 | #[tokio::test] 29 | async fn get_query_url_test() { 30 | let url = SubdomainCenter::get_query_url(TEST_DOMAIN); 31 | let expected = format!("{SUBDOMAINCENTER_URL}/?domain={TEST_DOMAIN}"); 32 | 33 | assert_eq!(url, expected); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_next_url_test() { 38 | let url = TEST_URL.parse().unwrap(); 39 | let next = SubdomainCenter::get_next_url(url, Content::Empty); 40 | 41 | assert!(next.is_none()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn extract_test() { 46 | let stub = "module/integrations/subdomaincenter.json"; 47 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 48 | 49 | let extracted = SubdomainCenter::extract(json, TEST_DOMAIN); 50 | let not_extracted = SubdomainCenter::extract(Value::Null, TEST_DOMAIN); 51 | 52 | assert!(extracted.is_ok()); 53 | assert!(not_extracted.is_err()); 54 | 55 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 56 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 57 | } 58 | -------------------------------------------------------------------------------- /src/modules/engines/bing.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Url; 2 | 3 | use crate::{ 4 | enums::dispatchers::{RequesterDispatcher, SubscanModuleDispatcher}, 5 | extractors::html::HTMLExtractor, 6 | modules::generics::engine::GenericSearchEngineModule, 7 | requesters::client::HTTPClient, 8 | types::core::SubscanModuleCoreComponents, 9 | }; 10 | 11 | pub const BING_MODULE_NAME: &str = "bing"; 12 | pub const BING_SEARCH_URL: &str = "https://www.bing.com/search"; 13 | pub const BING_SEARCH_PARAM: &str = "q"; 14 | pub const BING_CITE_TAG: &str = "cite"; 15 | 16 | /// Bing search engine enumerator 17 | /// 18 | /// It uses [`GenericSearchEngineModule`] its own inner 19 | /// here are the configurations 20 | /// 21 | /// | Property | Value | 22 | /// |:------------------:|:-----------------------------:| 23 | /// | Module Name | `bing` | 24 | /// | Search URL | | 25 | /// | Search Param | `q` | 26 | /// | Subdomain Selector | `cite` | 27 | /// | Requester | [`HTTPClient`] | 28 | /// | Extractor | [`HTMLExtractor`] | 29 | /// | Generic | [`GenericSearchEngineModule`] | 30 | pub struct Bing {} 31 | 32 | impl Bing { 33 | pub fn dispatcher() -> SubscanModuleDispatcher { 34 | let url = Url::parse(BING_SEARCH_URL); 35 | 36 | let extractor: HTMLExtractor = HTMLExtractor::new(BING_CITE_TAG.into(), vec![]); 37 | let requester: RequesterDispatcher = HTTPClient::default().into(); 38 | 39 | let generic = GenericSearchEngineModule { 40 | name: BING_MODULE_NAME.into(), 41 | param: BING_SEARCH_PARAM.into(), 42 | url: url.unwrap(), 43 | components: SubscanModuleCoreComponents { 44 | requester: requester.into(), 45 | extractor: extractor.into(), 46 | }, 47 | }; 48 | 49 | generic.into() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/modules/engines/google.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Url; 2 | 3 | use crate::{ 4 | enums::dispatchers::{RequesterDispatcher, SubscanModuleDispatcher}, 5 | extractors::html::HTMLExtractor, 6 | modules::generics::engine::GenericSearchEngineModule, 7 | requesters::client::HTTPClient, 8 | types::core::SubscanModuleCoreComponents, 9 | }; 10 | 11 | pub const GOOGLE_MODULE_NAME: &str = "google"; 12 | pub const GOOGLE_SEARCH_URL: &str = "https://www.google.com/search"; 13 | pub const GOOGLE_SEARCH_PARAM: &str = "q"; 14 | pub const GOOGLE_CITE_TAG: &str = "cite"; 15 | 16 | /// Google search engine enumerator 17 | /// 18 | /// It uses [`GenericSearchEngineModule`] its own inner 19 | /// here are the configurations 20 | /// 21 | /// | Property | Value | 22 | /// |:------------------:|:-------------------------------:| 23 | /// | Module Name | `google` | 24 | /// | Search URL | | 25 | /// | Search Param | `q` | 26 | /// | Subdomain Selector | `cite` | 27 | /// | Requester | [`HTTPClient`] | 28 | /// | Extractor | [`HTMLExtractor`] | 29 | /// | Generic | [`GenericSearchEngineModule`] | 30 | pub struct Google {} 31 | 32 | impl Google { 33 | pub fn dispatcher() -> SubscanModuleDispatcher { 34 | let url = Url::parse(GOOGLE_SEARCH_URL); 35 | 36 | let extractor: HTMLExtractor = HTMLExtractor::new(GOOGLE_CITE_TAG.into(), vec![]); 37 | let requester: RequesterDispatcher = HTTPClient::default().into(); 38 | 39 | let generic = GenericSearchEngineModule { 40 | name: GOOGLE_MODULE_NAME.into(), 41 | param: GOOGLE_SEARCH_PARAM.into(), 42 | url: url.unwrap(), 43 | components: SubscanModuleCoreComponents { 44 | requester: requester.into(), 45 | extractor: extractor.into(), 46 | }, 47 | }; 48 | 49 | generic.into() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/chaos_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, 6 | error::ModuleErrorKind::JSONExtract, 7 | interfaces::module::SubscanModuleInterface, 8 | modules::integrations::chaos::{Chaos, CHAOS_URL}, 9 | types::result::status::SubscanModuleStatus, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/chaos.json")] 20 | async fn run_test() { 21 | let mut chaos = Chaos::dispatcher(); 22 | let env_name = chaos.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "chaos-api-key"); 25 | funcs::wrap_module_url(&mut chaos, &stubr.path("/chaos")); 26 | 27 | let (results, status) = utils::run_module(chaos, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, SubscanModuleStatus::Finished); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = Chaos::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{CHAOS_URL}/{TEST_DOMAIN}/subdomains"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = TEST_URL.parse().unwrap(); 46 | let next = Chaos::get_next_url(url, Content::Empty); 47 | 48 | assert!(next.is_none()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn extract_test() { 53 | let stub = "module/integrations/chaos.json"; 54 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 55 | 56 | let extracted = Chaos::extract(json, TEST_DOMAIN); 57 | let not_extracted = Chaos::extract(Value::Null, TEST_DOMAIN); 58 | 59 | assert!(extracted.is_ok()); 60 | assert!(not_extracted.is_err()); 61 | 62 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 63 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 64 | } 65 | -------------------------------------------------------------------------------- /book/src/user-guide/commands/brute.md: -------------------------------------------------------------------------------- 1 | # `brute` 2 | 3 | With this command you can use the brute force technique to discover subdomains on a domain 4 | 5 | ## Argument List 6 | 7 | All arguments below can be used with the `brute` command, [see here](#common-use-cases) for common use cases 8 | 9 | | Name | Short | Description | 10 | | :----------------------- | :---: | :----------------------------------------------: | 11 | | `--domain` | `-d` | Target domain to be scanned | 12 | | `--wordlist` | `-w` | Wordlist file to be used during attack | 13 | | `--print` | | If sets, output will be logged on stdout | 14 | | `--stream-to-txt` | `-s` | Optional `txt` file to create file stream for the subdomains that found. If sets the `--output` parameter will be disabled | 15 | | `--output` | `-o` | Set output format (`txt`, `csv`, `json`, `html`) | 16 | | `--resolver-timeout` | | IP resolver timeout | 17 | | `--resolver-concurrency` | |  IP resolver concurrency level     | 18 | | `--resolver-list` | | A text file containing list of resolvers. See `resolverlist.template` | 19 | | `--help` | `-h` | Print help | 20 | 21 | ## Common Use Cases 22 | 23 | - Run a basic brute force attack with default settings 24 | 25 | ```bash 26 | ~$ subscan brute -d example.com -w wordlist.txt 27 | ``` 28 | 29 | - Increase resolver concurrency to improve attack speed 30 | 31 | ```bash 32 | ~$ subscan brute -d example.com -w wordlist.txt --resolver-concurrency 200 33 | ``` 34 | 35 | - Fine-tune IP address resolver component according to your network 36 | 37 | ```bash 38 | ~$ subscan brute -d example.com -w wordlist.txt --resolver-timeout 1 --resolver-concurrency 100 39 | ``` 40 | 41 | - Skip creating a report and print results directly to `stdout` 42 | 43 | ```bash 44 | ~$ subscan brute -d example.com -w wordlist.txt --print 45 | ``` 46 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/bevigil_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, 6 | error::ModuleErrorKind::JSONExtract, 7 | interfaces::module::SubscanModuleInterface, 8 | modules::integrations::bevigil::{Bevigil, BEVIGIL_URL}, 9 | types::result::status::SubscanModuleStatus, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/bevigil.json")] 20 | async fn run_test() { 21 | let mut bevigil = Bevigil::dispatcher(); 22 | let env_name = bevigil.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "bevigil-api-key"); 25 | funcs::wrap_module_url(&mut bevigil, &stubr.path("/bevigil")); 26 | 27 | let (results, status) = utils::run_module(bevigil, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, SubscanModuleStatus::Finished); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = Bevigil::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{BEVIGIL_URL}/{TEST_DOMAIN}/subdomains"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = TEST_URL.parse().unwrap(); 46 | let next = Bevigil::get_next_url(url, Content::Empty); 47 | 48 | assert!(next.is_none()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn extract_test() { 53 | let stub = "module/integrations/bevigil.json"; 54 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 55 | 56 | let extracted = Bevigil::extract(json, TEST_DOMAIN); 57 | let not_extracted = Bevigil::extract(Value::Null, TEST_DOMAIN); 58 | 59 | assert!(extracted.is_ok()); 60 | assert!(not_extracted.is_err()); 61 | 62 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 63 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 64 | } 65 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/netcraft_test.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Url; 2 | use subscan::{ 3 | enums::dispatchers::SubscanModuleDispatcher, 4 | modules::integrations::netcraft::{Netcraft, NETCRAFT_URL}, 5 | requesters::client::HTTPClient, 6 | types::result::status::SubscanModuleStatus, 7 | }; 8 | use tokio::sync::Mutex; 9 | 10 | use crate::common::{ 11 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 12 | mock::funcs, 13 | utils, 14 | }; 15 | 16 | #[tokio::test] 17 | #[stubr::mock("module/integrations/netcraft.json")] 18 | async fn run_test() { 19 | let mut netcraft = Netcraft::dispatcher(); 20 | let new_requester = HTTPClient::default(); 21 | 22 | funcs::wrap_module_url(&mut netcraft, &stubr.path("/netcraft")); 23 | 24 | if let SubscanModuleDispatcher::GenericIntegrationModule(ref mut netcraft) = netcraft { 25 | netcraft.components.requester = Mutex::new(new_requester.into()); 26 | } 27 | 28 | let (results, status) = utils::run_module(netcraft, TEST_DOMAIN).await; 29 | 30 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 31 | assert_eq!(status, SubscanModuleStatus::Finished); 32 | } 33 | 34 | #[tokio::test] 35 | async fn get_query_url_test() { 36 | let params = &[("restriction", "site+ends+with"), ("host", TEST_DOMAIN)]; 37 | 38 | let expected = Url::parse_with_params(NETCRAFT_URL, params).unwrap(); 39 | let url = Netcraft::get_query_url(TEST_DOMAIN); 40 | 41 | assert_eq!(url, expected.to_string()); 42 | } 43 | 44 | #[tokio::test] 45 | async fn get_next_url_test() { 46 | let html = "

"; 47 | 48 | let url = TEST_URL.parse().unwrap(); 49 | let next = Netcraft::get_next_url(url, html.into()); 50 | let expected = format!("{NETCRAFT_URL}/page-2"); 51 | 52 | assert_eq!(next.unwrap().to_string(), expected); 53 | } 54 | 55 | #[tokio::test] 56 | async fn get_next_url_fail_test() { 57 | let html = "

"; 58 | 59 | let url = TEST_URL.parse().unwrap(); 60 | let next = Netcraft::get_next_url(url, html.into()); 61 | 62 | assert!(next.is_none()); 63 | } 64 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/bufferover_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, 6 | error::ModuleErrorKind::JSONExtract, 7 | interfaces::module::SubscanModuleInterface, 8 | modules::integrations::bufferover::{BufferOver, BUFFEROVER_URL}, 9 | types::result::status::SubscanModuleStatus, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/bufferover.json")] 20 | async fn run_test() { 21 | let mut bufferover = BufferOver::dispatcher(); 22 | let env_name = bufferover.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "bufferover-api-key"); 25 | funcs::wrap_module_url(&mut bufferover, &stubr.path("/bufferover")); 26 | 27 | let (results, status) = utils::run_module(bufferover, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, SubscanModuleStatus::Finished); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = BufferOver::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{BUFFEROVER_URL}/dns?q={TEST_DOMAIN}"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = TEST_URL.parse().unwrap(); 46 | let next = BufferOver::get_next_url(url, Content::Empty); 47 | 48 | assert!(next.is_none()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn extract_test() { 53 | let stub = "module/integrations/bufferover.json"; 54 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 55 | 56 | let extracted = BufferOver::extract(json, TEST_DOMAIN); 57 | let not_extracted = BufferOver::extract(Value::Null, TEST_DOMAIN); 58 | 59 | assert!(extracted.is_ok()); 60 | assert!(not_extracted.is_err()); 61 | 62 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 63 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 64 | } 65 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/whoisxmlapi_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, 6 | error::ModuleErrorKind::JSONExtract, 7 | interfaces::module::SubscanModuleInterface, 8 | modules::integrations::whoisxmlapi::{WhoisXMLAPI, WHOISXMLAPI_URL}, 9 | types::result::status::SubscanModuleStatus, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/whoisxmlapi.json")] 20 | async fn run_test() { 21 | let mut whoisxmlapi = WhoisXMLAPI::dispatcher(); 22 | let env_name = whoisxmlapi.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "whoisxmlapi-api-key"); 25 | funcs::wrap_module_url(&mut whoisxmlapi, &stubr.path("/whoisxmlapi")); 26 | 27 | let (results, status) = utils::run_module(whoisxmlapi, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, SubscanModuleStatus::Finished); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = WhoisXMLAPI::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{WHOISXMLAPI_URL}/?domainName={TEST_DOMAIN}"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = TEST_URL.parse().unwrap(); 46 | let next = WhoisXMLAPI::get_next_url(url, Content::Empty); 47 | 48 | assert!(next.is_none()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn extract_test() { 53 | let stub = "module/integrations/whoisxmlapi.json"; 54 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 55 | 56 | let extracted = WhoisXMLAPI::extract(json, TEST_DOMAIN); 57 | let not_extracted = WhoisXMLAPI::extract(Value::Null, TEST_DOMAIN); 58 | 59 | assert!(extracted.is_ok()); 60 | assert!(not_extracted.is_err()); 61 | 62 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 63 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 64 | } 65 | -------------------------------------------------------------------------------- /src/constants.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | /// `Subscan` banner module path 4 | pub const SUBSCAN_BANNER_LOG_TARGET: &str = "subscan::banner"; 5 | /// `Subscan` environment variable namespace 6 | pub const SUBSCAN_ENV_NAMESPACE: &str = "SUBSCAN"; 7 | /// `Subscan` Chrome browser executable path env 8 | pub const SUBSCAN_CHROME_PATH_ENV: &str = "SUBSCAN_CHROME_PATH"; 9 | /// Concurrency level of module runner instances, count of threads 10 | pub const DEFAULT_MODULE_CONCURRENCY: u64 = 5; 11 | /// Concurrency level of resolver instances, count of threads 12 | pub const DEFAULT_RESOLVER_CONCURRENCY: u64 = 100; 13 | /// Default HTTP timeout as a [`Duration`] 14 | pub const DEFAULT_HTTP_TIMEOUT: Duration = Duration::from_secs(30); 15 | /// Default DNS resolver timeout as a [`Duration`] 16 | pub const DEFAULT_RESOLVER_TIMEOUT: Duration = Duration::from_millis(1000); 17 | /// Default User-Agent headers for HTTP requests 18 | pub const DEFAULT_USER_AGENT: &str = "\ 19 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) \ 20 | AppleWebKit/537.36 (KHTML, like Gecko) \ 21 | Chrome/135.0.0.0 Safari/537.36\ 22 | "; 23 | /// Asterisk character to indicate all modules 24 | pub const ASTERISK: &str = "*"; 25 | /// Time logging format 26 | pub const LOG_TIME_FORMAT: &str = "%H:%M:%S %Z"; 27 | /// Port number pattern to parse resolver ports from file 28 | pub const RL_PORT_PATTERN: &str = r#"(?([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))"#; 29 | /// IPv4 pattern to find IPv4 addresses from resolver list file. This regex 30 | /// pattern is used to parse IPv4 addresses and perform basic validations. 31 | /// It is used to make sure that IPv4 addresses are correctly written to 32 | /// the resolver list file in valid format 33 | pub const RL_IPV4_PATTERN: &str = r#"(?(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3})"#; 34 | /// IPv6 pattern to find IPv6 addresses from resolver list file. This regex 35 | /// pattern does not validate the IPv6 address, it only checks if it is 36 | /// written to the file in the valid format and is used to parse the IPv6 address 37 | pub const RL_IPV6_PATTERN: &str = r#"\[(?.+)\]"#; 38 | -------------------------------------------------------------------------------- /book/src/development/environment.md: -------------------------------------------------------------------------------- 1 | # Setup Development Environment 2 | 3 | This section covers topics like setting up a development environment and running tests for those who want to contribute to `Subscan` 4 | 5 | To set up your development environment, please follow the instructions below 6 | 7 | 1. Clone repository 8 | 9 | ```bash 10 | ~$ git clone https://github.com/eredotpkfr/subscan && cd subscan 11 | ``` 12 | 13 | 2. Install `pre-commit` and its hooks 14 | 15 | ```bash 16 | ~$ # Install pre-commit Mac or Linux 17 | ~$ make install-pre-commit-mac 18 | ~$ # Install pre-commit hooks 19 | ~$ make install-pre-commit-hooks 20 | ~$ # Check everything is OK 21 | ~$ pre-commit run -a 22 | ``` 23 | 24 | 3. Install required cargo tools for development 25 | 26 | ```bash 27 | ~$ # Install cargo tools 28 | ~$ make install-cargo-tools 29 | ``` 30 | 31 | 4. Create `.env` file from `.env.template` 32 | 33 | ```bash 34 | ~$ cp .env.template .env 35 | ``` 36 | 37 | 5. Finally build the project and run CLI 38 | 39 | ```bash 40 | ~$ cargo build && target/debug/subscan --help 41 | ``` 42 | 43 | ## Running Tests 44 | 45 | You have many options to run the tests, below are the command sets on how to run the tests differently 46 | 47 | ```bash 48 | ~$ # run all tests 49 | ~$ cargo test # or `make test` 50 | ~$ # capture outputs 51 | ~$ cargo test -- --nocapture 52 | ~$ # run only doc tests 53 | ~$ cargo test --doc 54 | ~$ # run a single test 55 | ~$ cargo test -- engines::bing_test::bing_run_test 56 | ~$ # run only integration tests 57 | ~$ cargo test --tests modules::integrations 58 | ``` 59 | 60 | To run tests via [nextest](https://nexte.st/), run following command 61 | 62 | ```bash 63 | ~$ make nextest 64 | ``` 65 | 66 | Create coverage report with [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov) 67 | 68 | ```bash 69 | ~$ make coverage 70 | ``` 71 | 72 | ## Building Docs 73 | 74 | To build documentations, run following command 75 | 76 | ```bash 77 | ~$ make doc # or `cargo doc` 78 | ``` 79 | 80 | To serve project book with hot reload, use following command 81 | 82 | ```bash 83 | ~$ # run book tests and serve 84 | ~$ make live-book 85 | ``` 86 | -------------------------------------------------------------------------------- /book/src/user-guide/environments.md: -------------------------------------------------------------------------------- 1 | # Environments 2 | 3 | `Subscan` has the ability to read all your environment variables from the `.env` file in your working directory. To learn how to define your environment variables in the `.env` file, you can refer to the `.env.template` file. All the `Subscan` environment variables uses `SUBSCAN` namespace as a prefix 4 | 5 | There are two types of environment variables: 6 | 7 | - **Dynamic:** These environment variables follow a specific format (e.g., `SUBSCAN__FOO`) and `Subscan` can read them automatically 8 | - **Static:** These are predefined environment variables that we know already 9 | 10 | ## Statics 11 | 12 | 13 | 14 | | Name | Required | Description | 15 | | :----------------------------- | :------: | :---------: | 16 | | `SUBSCAN_CHROME_PATH` | `false` | Specify your Chrome executable. If not specified, the Chrome binary will be fetched automatically by headless_chrome based on your system architecture | 17 | 18 | 19 | 20 | ## Dynamics 21 | 22 | | Name | Required | Description | 23 | | :----------------------------- | :------: | :---------: | 24 | | `SUBSCAN__HOST` | `false` | Some API integration modules can provide user specific host, for these cases, set module specific host | 25 | | `SUBSCAN__APIKEY` | `false` | Some modules may include API integration and require an API key for authentication. Set the API key in these cases | 26 | | `SUBSCAN__USERNAME` | `false` | Set the username for a module if it uses HTTP basic authentication | 27 | | `SUBSCAN__PASSWORD` | `false` | Set the password for a module if it uses HTTP basic authentication | 28 | 29 | ## Creating `.env` File 30 | 31 | Please see the [.env.template](https://github.com/eredotpkfr/subscan/blob/main/.env.template) file in project repository. Your `.env` file should follow a similar format as shown below 32 | 33 | ```bash 34 | SUBSCAN_BEVIGIL_APIKEY=foo 35 | SUBSCAN_BINARYEDGE_APIKEY=bar 36 | SUBSCAN_BUFFEROVER_APIKEY=baz 37 | ``` 38 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/dnsdumpsterapi_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, 6 | error::ModuleErrorKind::JSONExtract, 7 | interfaces::module::SubscanModuleInterface, 8 | modules::integrations::dnsdumpsterapi::{DNSDumpsterAPI, DNSDUMPSTERAPI_URL}, 9 | types::result::status::SubscanModuleStatus, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/dnsdumpsterapi.json")] 20 | async fn run_test() { 21 | let mut dnsdumpsterapi = DNSDumpsterAPI::dispatcher(); 22 | let env_name = dnsdumpsterapi.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "dnsdumpsterapi-api-key"); 25 | funcs::wrap_module_url(&mut dnsdumpsterapi, &stubr.path("/dnsdumpsterapi")); 26 | 27 | let (results, status) = utils::run_module(dnsdumpsterapi, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, SubscanModuleStatus::Finished); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = DNSDumpsterAPI::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{DNSDUMPSTERAPI_URL}/{TEST_DOMAIN}"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = TEST_URL.parse().unwrap(); 46 | let next = DNSDumpsterAPI::get_next_url(url, Content::Empty); 47 | 48 | assert!(next.is_none()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn extract_test() { 53 | let stub = "module/integrations/dnsdumpsterapi.json"; 54 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 55 | 56 | let extracted = DNSDumpsterAPI::extract(json, TEST_DOMAIN); 57 | let not_extracted = DNSDumpsterAPI::extract(Value::Null, TEST_DOMAIN); 58 | 59 | assert!(extracted.is_ok()); 60 | assert!(not_extracted.is_err()); 61 | 62 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 63 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 64 | } 65 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/securitytrails_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use serde_json::Value; 4 | use subscan::{ 5 | enums::content::Content, 6 | error::ModuleErrorKind::JSONExtract, 7 | interfaces::module::SubscanModuleInterface, 8 | modules::integrations::securitytrails::{SecurityTrails, SECURITYTRAILS_URL}, 9 | types::result::status::SubscanModuleStatus, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/securitytrails.json")] 20 | async fn run_test() { 21 | let mut securitytrails = SecurityTrails::dispatcher(); 22 | let env_name = securitytrails.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "securitytrails-api-key"); 25 | funcs::wrap_module_url(&mut securitytrails, &stubr.path("/securitytrails")); 26 | 27 | let (results, status) = utils::run_module(securitytrails, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, SubscanModuleStatus::Finished); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = SecurityTrails::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{SECURITYTRAILS_URL}/{TEST_DOMAIN}/subdomains"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = TEST_URL.parse().unwrap(); 46 | let next = SecurityTrails::get_next_url(url, Content::Empty); 47 | 48 | assert!(next.is_none()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn extract_test() { 53 | let stub = "module/integrations/securitytrails.json"; 54 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 55 | 56 | let extracted = SecurityTrails::extract(json, TEST_DOMAIN); 57 | let not_extracted = SecurityTrails::extract(Value::Null, TEST_DOMAIN); 58 | 59 | assert!(extracted.is_ok()); 60 | assert!(not_extracted.is_err()); 61 | 62 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 63 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 64 | } 65 | -------------------------------------------------------------------------------- /src/modules/integrations/mod.rs: -------------------------------------------------------------------------------- 1 | /// `AlienVault` API integration module 2 | pub mod alienvault; 3 | /// `Anubis` API integration module 4 | pub mod anubis; 5 | /// `Bevigil` API integration module, API key required 6 | pub mod bevigil; 7 | /// `BinaryEdge` API integration mmodule, API key required 8 | pub mod binaryedge; 9 | /// `BufferOver` API integration mmodule, API key required 10 | pub mod bufferover; 11 | /// `BuiltWith` API integration mmodule, API key required 12 | pub mod builtwith; 13 | /// `Censys` API integration, basic HTTP auth required but `Authorization` 14 | /// header can be used (e.g. `Authorization: Basic foo`) 15 | pub mod censys; 16 | /// `CertSpotter` API integration, API key required 17 | pub mod certspotter; 18 | /// `Chaos` API integration, API key required 19 | pub mod chaos; 20 | /// `CommonCrawl` non-generic module integration 21 | pub mod commoncrawl; 22 | /// `Crt.sh` API integration 23 | pub mod crtsh; 24 | /// `Digitorus` HTML crawler integration 25 | pub mod digitorus; 26 | /// `DNSDumpster` API integration, API key required 27 | pub mod dnsdumpsterapi; 28 | /// `DNSDumpster` non-generic integration 29 | pub mod dnsdumpstercrawler; 30 | /// `DnsRepo` html crawler integration 31 | pub mod dnsrepo; 32 | /// `GitHub` non-generic integration 33 | pub mod github; 34 | /// `HackerTarget` HTML crawler integration 35 | pub mod hackertarget; 36 | /// `Leakix` API integration 37 | pub mod leakix; 38 | /// `Netcraft` HTML crawler integration 39 | pub mod netcraft; 40 | /// `Netlas` non-generic API integration, API key required 41 | pub mod netlas; 42 | /// `SecurityTrails` API integration, API key required 43 | pub mod securitytrails; 44 | /// `Shodan` API integration, API key required 45 | pub mod shodan; 46 | /// `Sitedossier` HTML crawler integration 47 | pub mod sitedossier; 48 | /// `SubdomainCenter` API integration module 49 | pub mod subdomaincenter; 50 | /// `ThreatCrowd` API integration 51 | pub mod threatcrowd; 52 | /// `VirusTotal` API integration, API key required 53 | pub mod virustotal; 54 | /// `WaybackArchive` non-generic integration 55 | pub mod waybackarchive; 56 | /// `WhoisXMLAPI` API integration, API key required 57 | pub mod whoisxmlapi; 58 | /// `ZoomEye` API integration, API key required 59 | pub mod zoomeye; 60 | -------------------------------------------------------------------------------- /src/modules/engines/yahoo.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Url; 2 | 3 | use crate::{ 4 | enums::dispatchers::{RequesterDispatcher, SubscanModuleDispatcher}, 5 | extractors::html::HTMLExtractor, 6 | modules::generics::engine::GenericSearchEngineModule, 7 | requesters::client::HTTPClient, 8 | types::core::SubscanModuleCoreComponents, 9 | }; 10 | 11 | pub const YAHOO_MODULE_NAME: &str = "yahoo"; 12 | pub const YAHOO_SEARCH_URL: &str = "https://search.yahoo.com/search"; 13 | pub const YAHOO_SEARCH_PARAM: &str = "p"; 14 | pub const YAHOO_CITE_TAG: &str = "ol > li > div > div > h3 > a > span"; 15 | 16 | /// Yahoo search engine enumerator 17 | /// 18 | /// It uses [`GenericSearchEngineModule`] its own inner 19 | /// here are the configurations 20 | /// 21 | /// | Property | Value | 22 | /// |:------------------:|:-------------------------------------:| 23 | /// | Module Name | `yahoo` | 24 | /// | Search URL | | 25 | /// | Search Param | `p` | 26 | /// | Subdomain Selector | `ol > li > div > div > h3 > a > span` | 27 | /// | Requester | [`HTTPClient`] | 28 | /// | Extractor | [`HTMLExtractor`] | 29 | /// | Generic | [`GenericSearchEngineModule`] | 30 | pub struct Yahoo {} 31 | 32 | impl Yahoo { 33 | pub fn dispatcher() -> SubscanModuleDispatcher { 34 | let url = Url::parse(YAHOO_SEARCH_URL); 35 | 36 | let selector: String = YAHOO_CITE_TAG.into(); 37 | let removes: Vec = vec!["".into(), "".into()]; 38 | 39 | let extractor: HTMLExtractor = HTMLExtractor::new(selector, removes); 40 | let requester: RequesterDispatcher = HTTPClient::default().into(); 41 | 42 | let generic = GenericSearchEngineModule { 43 | name: YAHOO_MODULE_NAME.into(), 44 | param: YAHOO_SEARCH_PARAM.into(), 45 | url: url.unwrap(), 46 | components: SubscanModuleCoreComponents { 47 | requester: requester.into(), 48 | extractor: extractor.into(), 49 | }, 50 | }; 51 | 52 | generic.into() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /book/src/development/generics/integration.md: -------------------------------------------------------------------------------- 1 | # Generic Integration Module 2 | 3 | The [`GenericIntegrationModule`](https://docs.rs/subscan/latest/subscan/modules/generics/integration/struct.GenericIntegrationModule.html) is primarily used for simple `API` integrations. To understand how it works, check the source code on the [docs.rs](https://docs.rs/subscan/latest/subscan/modules/generics/integration/struct.GenericIntegrationModule.html) page. Additionally, looking at the source code of other modules that use this implementation will help you understand how to utilize it 4 | 5 | A module that uses this one internally would look like the following 6 | 7 | ```rust,ignore 8 | pub const EXAMPLE_MODULE_NAME: &str = "example"; 9 | pub const EXAMPLE_URL: &str = "https://api.example.com/api/v1"; 10 | 11 | pub struct ExampleModule {} 12 | 13 | impl ExampleModule { 14 | pub fn dispatcher() -> SubscanModuleDispatcher { 15 | let requester: RequesterDispatcher = HTTPClient::default().into(); 16 | let extractor: JSONExtractor = JSONExtractor::new(Box::new(Self::extract)); 17 | 18 | let generic = GenericIntegrationModule { 19 | name: EXAMPLE_MODULE_NAME.into(), 20 | auth: AuthenticationMethod::NoAuthentication, 21 | funcs: GenericIntegrationCoreFuncs { 22 | url: Box::new(Self::get_query_url), 23 | next: Box::new(Self::get_next_url), 24 | }, 25 | components: SubscanModuleCoreComponents { 26 | requester: requester.into(), 27 | extractor: extractor.into(), 28 | }, 29 | }; 30 | 31 | generic.into() 32 | } 33 | 34 | pub fn get_query_url(domain: &str) -> String { 35 | format!("{EXAMPLE_URL}/{domain}/subdomains") 36 | } 37 | 38 | pub fn get_next_url(_url: Url, _content: Content) -> Option { 39 | None 40 | } 41 | 42 | pub fn extract(content: Value, _domain: &str) -> Result> { 43 | if let Some(items) = content["items"].as_array() { 44 | let filter = |item: &Value| Some(item["hostname"].as_str()?.to_string()); 45 | 46 | return Ok(items.iter().filter_map(filter).collect()); 47 | } 48 | 49 | Err(JSONExtract.into()) 50 | } 51 | } 52 | ``` 53 | -------------------------------------------------------------------------------- /src/types/result/statistics.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use chrono::{serde::ts_seconds, DateTime, TimeDelta, Utc}; 4 | use serde::Serialize; 5 | 6 | use super::status::{SkipReason::SkippedByUser, SubscanModuleStatus}; 7 | use crate::utilities::serializers::td_to_seconds; 8 | 9 | /// Subscan result statistics data type 10 | pub type SubscanResultStatistics = HashMap; 11 | 12 | /// Stores single [`SubscanModule`](crate::types::core::SubscanModule) statistics 13 | #[derive(Clone, Debug, Serialize)] 14 | pub struct SubscanModuleStatistic { 15 | pub status: SubscanModuleStatus, 16 | pub count: usize, 17 | #[serde(with = "ts_seconds")] 18 | pub started_at: DateTime, 19 | #[serde(with = "ts_seconds")] 20 | pub finished_at: DateTime, 21 | #[serde(serialize_with = "td_to_seconds")] 22 | pub elapsed: TimeDelta, 23 | } 24 | 25 | impl Default for SubscanModuleStatistic { 26 | fn default() -> Self { 27 | Self { 28 | status: SubscanModuleStatus::Started, 29 | count: 0, 30 | started_at: Utc::now(), 31 | finished_at: Utc::now(), 32 | elapsed: TimeDelta::zero(), 33 | } 34 | } 35 | } 36 | 37 | impl SubscanModuleStatistic { 38 | /// Create skipped module statistics 39 | /// 40 | /// # Examples 41 | /// 42 | /// ``` 43 | /// use subscan::types::result::{ 44 | /// statistics::SubscanModuleStatistic, 45 | /// status::SkipReason::SkippedByUser, 46 | /// }; 47 | /// 48 | /// let skipped = SubscanModuleStatistic::skipped(); 49 | /// 50 | /// assert_eq!(skipped.status, SkippedByUser.into()); 51 | /// assert_eq!(skipped.count, 0); 52 | /// assert_eq!(skipped.elapsed.num_seconds(), 0); 53 | /// ``` 54 | pub fn skipped() -> Self { 55 | Self { 56 | status: SkippedByUser.into(), 57 | count: 0, 58 | started_at: Utc::now(), 59 | finished_at: Utc::now(), 60 | elapsed: TimeDelta::zero(), 61 | } 62 | } 63 | 64 | pub async fn finish_with_status(&mut self, status: SubscanModuleStatus) { 65 | self.finished_at = Utc::now(); 66 | self.status = status; 67 | self.elapsed = self.finished_at - self.started_at; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/censys_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use reqwest::Url; 4 | use serde_json::{json, Value}; 5 | use subscan::{ 6 | enums::content::Content, 7 | error::ModuleErrorKind::JSONExtract, 8 | interfaces::module::SubscanModuleInterface, 9 | modules::integrations::censys::{Censys, CENSYS_URL}, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/censys.json")] 20 | async fn run_test() { 21 | let mut censys = Censys::dispatcher(); 22 | let env_name = censys.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "censys-api-key"); 25 | funcs::wrap_module_url(&mut censys, &stubr.path("/censys")); 26 | 27 | let (results, status) = utils::run_module(censys, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, JSONExtract.into()); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = Censys::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{CENSYS_URL}?q={TEST_DOMAIN}"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = Url::parse(TEST_URL).unwrap(); 46 | let json = json!({"result": {"links": {"next": "foo"}}}); 47 | let expected = Url::parse(&format!("{TEST_URL}/?cursor=foo")).unwrap(); 48 | 49 | let mut next = Censys::get_next_url(url.clone(), Content::Empty); 50 | 51 | assert!(next.is_none()); 52 | 53 | next = Censys::get_next_url(url, json.into()); 54 | 55 | assert_eq!(next.unwrap(), expected); 56 | } 57 | 58 | #[tokio::test] 59 | async fn extract_test() { 60 | let stub = "module/integrations/censys.json"; 61 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 62 | 63 | let extracted = Censys::extract(json, TEST_DOMAIN); 64 | let not_extracted = Censys::extract(Value::Null, TEST_DOMAIN); 65 | 66 | assert!(extracted.is_ok()); 67 | assert!(not_extracted.is_err()); 68 | 69 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 70 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 71 | } 72 | -------------------------------------------------------------------------------- /src/types/config/pool.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::{File, OpenOptions}, 3 | path::PathBuf, 4 | sync::RwLock, 5 | }; 6 | 7 | use super::subscan::SubscanConfig; 8 | use crate::enums::cache::CacheFilter; 9 | 10 | #[derive(Clone, Default)] 11 | pub struct PoolConfig { 12 | pub concurrency: u64, 13 | pub filter: CacheFilter, 14 | pub print: bool, 15 | pub stream: Option, 16 | } 17 | 18 | impl From for PoolConfig { 19 | /// Create [`PoolConfig`] from [`SubscanConfig`] 20 | /// 21 | /// # Examples 22 | /// 23 | /// ``` 24 | /// use subscan::types::config::{pool::PoolConfig, subscan::SubscanConfig}; 25 | /// 26 | /// let sconfig = SubscanConfig::default(); 27 | /// let config = PoolConfig::from(sconfig.clone()); 28 | /// 29 | /// assert_eq!(sconfig.concurrency, config.concurrency); 30 | /// assert_eq!(sconfig.filter, config.filter); 31 | /// assert_eq!(sconfig.print, config.print); 32 | /// assert_eq!(sconfig.stream, config.stream); 33 | /// ``` 34 | fn from(config: SubscanConfig) -> Self { 35 | Self { 36 | concurrency: config.concurrency, 37 | filter: config.filter, 38 | print: config.print, 39 | stream: config.stream, 40 | } 41 | } 42 | } 43 | 44 | impl PoolConfig { 45 | /// Get stream file descriptor 46 | /// 47 | /// # Examples 48 | /// 49 | /// ``` 50 | /// use subscan::types::config::{pool::PoolConfig, subscan::SubscanConfig}; 51 | /// 52 | /// #[tokio::main] 53 | /// async fn main() { 54 | /// let sconfig = SubscanConfig::default(); 55 | /// let config = PoolConfig::from(sconfig.clone()); 56 | /// 57 | /// assert!(config.get_stream_file().await.is_none()); 58 | /// } 59 | /// ``` 60 | pub async fn get_stream_file(&self) -> Option> { 61 | self.stream.as_ref().map(|path| { 62 | RwLock::new( 63 | OpenOptions::new() 64 | .create(true) 65 | .append(true) 66 | .read(true) 67 | .truncate(false) 68 | .open(path) 69 | .unwrap_or_else(|_| panic!("Cannot create {} file!", path.to_str().unwrap())), 70 | ) 71 | }) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /book/src/development/components/extractors.md: -------------------------------------------------------------------------------- 1 | # Extractors 2 | 3 | Extractor components are responsible for parsing subdomain addresses from any `Content` object 4 | 5 | The extractor components already implemented in `Subscan` are as follows 6 | 7 | - [HTMLExtractor](https://docs.rs/subscan/latest/subscan/extractors/html/struct.HTMLExtractor.html) 8 | 9 | Extracts subdomain addresses from inner text by given `XPath` or `CSS` selector 10 | 11 | - [JSONExtractor](https://docs.rs/subscan/latest/subscan/extractors/json/struct.JSONExtractor.html) 12 | 13 | Extracts subdomain addresses from `JSON` content. `JSON` parsing function must be given for this extractor 14 | 15 | - [RegexExtractor](https://docs.rs/subscan/latest/subscan/extractors/regex/struct.RegexExtractor.html) 16 | 17 | Regex extractor component generates subdomain pattern by given domain address and extracts subdomains via this pattern 18 | 19 | ## Create Your Custom Extractor 20 | 21 | Each extractor component should be implemented following the interface below. For a better understanding, you can explore the [docs.rs](https://docs.rs/subscan/latest/subscan/interfaces/extractor/index.html) page and review the crates listed below 22 | 23 | - [`async_trait`](https://github.com/dtolnay/async-trait) 24 | - [`enum_dispatch`](https://gitlab.com/antonok/enum_dispatch) 25 | 26 | ```rust,ignore 27 | #[async_trait] 28 | #[enum_dispatch] 29 | pub trait SubdomainExtractorInterface: Send + Sync { 30 | // Generic extract method, it should extract subdomain addresses 31 | // from given Content 32 | async fn extract(&self, content: Content, domain: &str) -> Result>; 33 | } 34 | ``` 35 | 36 | Below is a simple example of a custom extractor. For more examples, you can check the [examples/](https://github.com/eredotpkfr/subscan/tree/main/examples) folder on the project's GitHub page. You can also refer to the source code of predefined requester implementations for a better understanding 37 | 38 | ```rust,ignore 39 | pub struct CustomExtractor {} 40 | 41 | #[async_trait] 42 | impl SubdomainExtractorInterface for CustomExtractor { 43 | async fn extract(&self, content: Content, _domain: &str) -> Result> { 44 | let subdomain = content.as_string().replace("-", ""); 45 | 46 | Ok([subdomain].into()) 47 | } 48 | } 49 | ``` 50 | -------------------------------------------------------------------------------- /tests/components/common/mock/modules.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use reqwest::Url; 4 | use serde_json::Value; 5 | use subscan::{ 6 | enums::{auth::AuthenticationMethod, dispatchers::RequesterDispatcher}, 7 | error::ModuleErrorKind::JSONExtract, 8 | extractors::{json::JSONExtractor, regex::RegexExtractor}, 9 | modules::generics::{engine::GenericSearchEngineModule, integration::GenericIntegrationModule}, 10 | requesters::client::HTTPClient, 11 | types::{ 12 | core::SubscanModuleCoreComponents, func::GenericIntegrationCoreFuncs, 13 | query::SearchQueryParam, 14 | }, 15 | }; 16 | 17 | use super::funcs::wrap_url_with_mock_func; 18 | use crate::common::utils::current_thread_hex; 19 | 20 | pub fn generic_search_engine(url: &str) -> GenericSearchEngineModule { 21 | let requester = RequesterDispatcher::HTTPClient(HTTPClient::default()); 22 | let extractor = RegexExtractor::default(); 23 | let url = Url::parse(url); 24 | 25 | GenericSearchEngineModule { 26 | name: current_thread_hex(), 27 | url: url.unwrap(), 28 | param: SearchQueryParam::from("q"), 29 | components: SubscanModuleCoreComponents { 30 | requester: requester.into(), 31 | extractor: extractor.into(), 32 | }, 33 | } 34 | } 35 | 36 | pub fn generic_integration(url: &str, auth: AuthenticationMethod) -> GenericIntegrationModule { 37 | let parse = |json: Value, _domain: &str| { 38 | if let Some(subdomains) = json["subdomains"].as_array() { 39 | let filter = |item: &Value| Some(item.as_str()?.to_string()); 40 | 41 | return Ok(BTreeSet::from_iter(subdomains.iter().filter_map(filter))); 42 | } 43 | 44 | Err(JSONExtract.into()) 45 | }; 46 | 47 | let requester = RequesterDispatcher::HTTPClient(HTTPClient::default()); 48 | let extractor = JSONExtractor::new(Box::new(parse)); 49 | 50 | GenericIntegrationModule { 51 | name: current_thread_hex(), 52 | auth, 53 | funcs: GenericIntegrationCoreFuncs { 54 | url: wrap_url_with_mock_func(url), 55 | next: Box::new(|_, _| None), 56 | }, 57 | components: SubscanModuleCoreComponents { 58 | requester: requester.into(), 59 | extractor: extractor.into(), 60 | }, 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "subscan" 3 | version = "1.3.0" 4 | edition = "2021" 5 | description = "A subdomain enumeration tool leveraging diverse techniques, designed for advanced pentesting operations" 6 | documentation = "https://docs.rs/subscan" 7 | homepage = "https://www.erdoganyoksul.com/subscan" 8 | readme = "README.md" 9 | authors = ["Erdoğan YOKSUL "] 10 | repository = "https://github.com/eredotpkfr/subscan" 11 | license-file = "LICENSE" 12 | keywords = [ 13 | "pentesting-tool", 14 | "subdomain-finder", 15 | "bruteforce", 16 | "zonetransfer", 17 | "searchengines", 18 | ] 19 | 20 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 21 | 22 | [dependencies] 23 | async-trait = "0.1.89" 24 | chrono = { version = "0.4.42", features = ["serde"] } 25 | clap = { version = "4.5.53", features = ["derive"] } 26 | clap-verbosity-flag = "3.0.4" 27 | colog = "1.3.0" 28 | colored = "3.0.0" 29 | csv = "1.4.0" 30 | derive_more = { version = "2.0.1", features = ["deref", "display", "from"] } 31 | dotenvy = "0.15.7" 32 | enum_dispatch = "0.3.13" 33 | env_logger = "0.11.8" 34 | flume = "0.11.1" 35 | futures = "0.3.31" 36 | headless_chrome = { version = "1.0.18", default-features = false, features = ["fetch"] } 37 | hickory-client = { version = "0.25.2", default-features = false } 38 | hickory-resolver = { version = "0.25.2", default-features = false, features = ["system-config", "tokio"] } 39 | itertools = "0.14.0" 40 | log = "0.4.28" 41 | prettytable-rs = "0.10.0" 42 | regex = "1.12.2" 43 | reqwest = { version = "0.12.24", features = ["json", "stream"] } 44 | scraper = "0.24.0" 45 | serde = { version = "1.0.228", features = ["derive"] } 46 | serde_json = "1.0.145" 47 | tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "tokio-macros"] } 48 | tokio-util = "0.7.17" 49 | url = "2.5.4" 50 | 51 | [dev-dependencies] 52 | automod = "1.0.15" 53 | hickory-server = "0.25.2" 54 | md5 = "0.8.0" 55 | stubr = "0.6.2" 56 | tempfile = "3.23.0" 57 | 58 | [profile.release] 59 | lto = true 60 | strip = true 61 | opt-level = "z" 62 | codegen-units = 1 63 | panic = "abort" 64 | 65 | # The profile that 'dist' will build with 66 | [profile.dist] 67 | inherits = "release" 68 | 69 | [profile.coverage-ci] 70 | inherits = "release" 71 | 72 | [package.metadata.cargo-machete] 73 | ignored = ["prettytable-rs"] 74 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/certspotter_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use reqwest::Url; 4 | use serde_json::Value; 5 | use subscan::{ 6 | enums::content::Content, 7 | error::ModuleErrorKind::JSONExtract, 8 | interfaces::module::SubscanModuleInterface, 9 | modules::integrations::certspotter::{CertSpotter, CERTSPOTTER_URL}, 10 | types::result::status::SubscanModuleStatus, 11 | }; 12 | 13 | use crate::common::{ 14 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 15 | mock::funcs, 16 | utils, 17 | }; 18 | 19 | #[tokio::test] 20 | #[stubr::mock("module/integrations/certspotter.json")] 21 | async fn run_test() { 22 | let mut certspotter = CertSpotter::dispatcher(); 23 | let env_name = certspotter.envs().await.apikey.name; 24 | 25 | env::set_var(&env_name, "certspotter-api-key"); 26 | funcs::wrap_module_url(&mut certspotter, &stubr.path("/certspotter")); 27 | 28 | let (results, status) = utils::run_module(certspotter, TEST_DOMAIN).await; 29 | 30 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 31 | assert_eq!(status, SubscanModuleStatus::Finished); 32 | 33 | env::remove_var(env_name); 34 | } 35 | 36 | #[tokio::test] 37 | async fn get_query_url_test() { 38 | let url = CertSpotter::get_query_url(TEST_DOMAIN); 39 | 40 | let params = &[ 41 | ("domain", TEST_DOMAIN), 42 | ("include_subdomains", "true"), 43 | ("expand", "dns_names"), 44 | ]; 45 | 46 | let expected = Url::parse_with_params(CERTSPOTTER_URL, params); 47 | 48 | assert_eq!(url, expected.unwrap().to_string()); 49 | } 50 | 51 | #[tokio::test] 52 | async fn get_next_url_test() { 53 | let url = TEST_URL.parse().unwrap(); 54 | let next = CertSpotter::get_next_url(url, Content::Empty); 55 | 56 | assert!(next.is_none()); 57 | } 58 | 59 | #[tokio::test] 60 | async fn extract_test() { 61 | let stub = "module/integrations/certspotter.json"; 62 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 63 | 64 | let extracted = CertSpotter::extract(json, TEST_DOMAIN); 65 | let not_extracted = CertSpotter::extract(Value::Null, TEST_DOMAIN); 66 | 67 | assert!(extracted.is_ok()); 68 | assert!(not_extracted.is_err()); 69 | 70 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 71 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 72 | } 73 | -------------------------------------------------------------------------------- /tests/components/modules/integrations/binaryedge_test.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | use reqwest::Url; 4 | use serde_json::Value; 5 | use subscan::{ 6 | enums::content::Content, 7 | error::ModuleErrorKind::JSONExtract, 8 | interfaces::module::SubscanModuleInterface, 9 | modules::integrations::binaryedge::{BinaryEdge, BINARYEDGE_URL}, 10 | }; 11 | 12 | use crate::common::{ 13 | constants::{TEST_BAR_SUBDOMAIN, TEST_DOMAIN, TEST_URL}, 14 | mock::funcs, 15 | utils, 16 | }; 17 | 18 | #[tokio::test] 19 | #[stubr::mock("module/integrations/binaryedge.json")] 20 | async fn run_test() { 21 | let mut binaryedge = BinaryEdge::dispatcher(); 22 | let env_name = binaryedge.envs().await.apikey.name; 23 | 24 | env::set_var(&env_name, "binaryedge-api-key"); 25 | funcs::wrap_module_url(&mut binaryedge, &stubr.path("/binaryedge")); 26 | 27 | let (results, status) = utils::run_module(binaryedge, TEST_DOMAIN).await; 28 | 29 | assert_eq!(results, [TEST_BAR_SUBDOMAIN.into()].into()); 30 | assert_eq!(status, JSONExtract.into()); 31 | 32 | env::remove_var(env_name); 33 | } 34 | 35 | #[tokio::test] 36 | async fn get_query_url_test() { 37 | let url = BinaryEdge::get_query_url(TEST_DOMAIN); 38 | let expected = format!("{BINARYEDGE_URL}/{TEST_DOMAIN}"); 39 | 40 | assert_eq!(url, expected); 41 | } 42 | 43 | #[tokio::test] 44 | async fn get_next_url_test() { 45 | let url = Url::parse(TEST_URL).unwrap(); 46 | 47 | let mut next = BinaryEdge::get_next_url(url.clone(), Content::Empty).unwrap(); 48 | let mut expected = Url::parse(&format!("{TEST_URL}/?page=2")).unwrap(); 49 | 50 | assert_eq!(next, expected); 51 | 52 | next = BinaryEdge::get_next_url(next, Content::Empty).unwrap(); 53 | expected = Url::parse(&format!("{TEST_URL}/?page=3")).unwrap(); 54 | 55 | assert_eq!(next, expected); 56 | } 57 | 58 | #[tokio::test] 59 | async fn extract_test() { 60 | let stub = "module/integrations/binaryedge.json"; 61 | let json = utils::read_stub(stub)["response"]["jsonBody"].clone(); 62 | 63 | let extracted = BinaryEdge::extract(json, TEST_DOMAIN); 64 | let not_extracted = BinaryEdge::extract(Value::Null, TEST_DOMAIN); 65 | 66 | assert!(extracted.is_ok()); 67 | assert!(not_extracted.is_err()); 68 | 69 | assert_eq!(extracted.unwrap(), [TEST_BAR_SUBDOMAIN.into()].into()); 70 | assert_eq!(not_extracted.err().unwrap(), JSONExtract.into()); 71 | } 72 | -------------------------------------------------------------------------------- /examples/custom_module.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeSet; 2 | 3 | use async_trait::async_trait; 4 | use flume::Sender; 5 | use subscan::{ 6 | enums::{ 7 | dispatchers::{RequesterDispatcher, SubdomainExtractorDispatcher}, 8 | result::{OptionalSubscanModuleResult, SubscanModuleResult}, 9 | }, 10 | extractors::regex::RegexExtractor, 11 | interfaces::module::SubscanModuleInterface, 12 | requesters::client::HTTPClient, 13 | types::{core::Subdomain, result::item::SubscanModuleResultItem}, 14 | }; 15 | use tokio::sync::Mutex; 16 | 17 | pub struct CustomModule { 18 | pub requester: Mutex, 19 | pub extractor: SubdomainExtractorDispatcher, 20 | } 21 | 22 | #[async_trait] 23 | impl SubscanModuleInterface for CustomModule { 24 | async fn name(&self) -> &str { 25 | &"name" 26 | } 27 | 28 | async fn requester(&self) -> Option<&Mutex> { 29 | Some(&self.requester) 30 | } 31 | 32 | async fn extractor(&self) -> Option<&SubdomainExtractorDispatcher> { 33 | Some(&self.extractor) 34 | } 35 | 36 | async fn run(&mut self, _domain: &str, results: Sender) { 37 | let subdomains = BTreeSet::from_iter([Subdomain::from("bar.foo.com")]); 38 | 39 | for subdomain in &subdomains { 40 | results.send((self.name().await, subdomain).into()).unwrap(); 41 | } 42 | } 43 | } 44 | 45 | #[tokio::main] 46 | async fn main() { 47 | let requester: RequesterDispatcher = HTTPClient::default().into(); 48 | let extracator: RegexExtractor = RegexExtractor::default(); 49 | 50 | let (tx, rx) = flume::unbounded::(); 51 | 52 | let mut module = CustomModule { 53 | requester: requester.into(), 54 | extractor: extracator.into(), 55 | }; 56 | 57 | assert!(module.requester().await.is_some()); 58 | assert!(module.extractor().await.is_some()); 59 | 60 | assert_eq!(module.name().await, "name"); 61 | 62 | module.run("foo.com", tx).await; 63 | 64 | let result = rx.recv().unwrap(); 65 | let item = SubscanModuleResultItem { 66 | module: "name".into(), 67 | subdomain: Subdomain::from("bar.foo.com"), 68 | }; 69 | let expected = &SubscanModuleResult::SubscanModuleResultItem(item); 70 | 71 | assert!(result.is_some()); 72 | assert_eq!(result.as_ref().unwrap(), expected); 73 | } 74 | --------------------------------------------------------------------------------