├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md └── src └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hostparser" 3 | authors = ["zoid", ""] 4 | description = "A very fast hostparser." 5 | version = "0.1.3" 6 | edition = "2021" 7 | license = "MIT" 8 | repository = "https://github.com/ethicalhackingplayground/hostparser" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [dependencies] 13 | governor = "0.5.1" 14 | itertools = "0.10.5" 15 | futures = "0.3.26" 16 | futures-util = "0.3.26" 17 | tokio = { version = "^1.2.0", features = ["full"] } 18 | spmc = "0.3.0" 19 | clap = { version = "^3.0.0-beta.2" } 20 | tldextract = "0.6.0" 21 | reqwest = { version = "0.11.14", features = ["native-tls", "blocking"] } 22 | async-std = "1.12.0" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 zoidsec 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hostparser 2 | A very fast hostparser written in rust.🦀 3 | 4 | ### Installation 5 | 6 | ```bash 7 | cargo install hostparser 8 | ``` 9 | 10 | ### Usage 11 | 12 | ```bash 13 | echo "glass-eur.ext.google.com" | hostparser 14 | ``` 15 | 16 | #### OUTPUT 17 | ``` 18 | google.com 19 | ``` 20 | 21 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use async_std::io; 2 | use async_std::io::prelude::*; 3 | use clap::{App, Arg}; 4 | use futures::{stream::FuturesUnordered, StreamExt}; 5 | use governor::{Quota, RateLimiter}; 6 | use std::error::Error; 7 | use tldextract::{TldExtractor, TldOption}; 8 | use tokio::{runtime::Builder, task}; 9 | 10 | #[derive(Clone, Debug)] 11 | pub struct Job { 12 | host: Option, 13 | } 14 | 15 | #[derive(Clone, Debug)] 16 | pub struct JobResult { 17 | pub data: String, 18 | } 19 | 20 | #[tokio::main] 21 | async fn main() -> Result<(), Box> { 22 | // parse the cli arguments 23 | let matches = App::new("hostparser") 24 | .version("0.1.3") 25 | .author("Blake Jacobs ") 26 | .about("A very fast hostparser") 27 | .arg( 28 | Arg::with_name("rate") 29 | .short('r') 30 | .long("rate") 31 | .takes_value(true) 32 | .default_value("1000") 33 | .display_order(2) 34 | .help("Maximum in-flight requests per second"), 35 | ) 36 | .arg( 37 | Arg::with_name("concurrency") 38 | .short('c') 39 | .long("concurrency") 40 | .default_value("100") 41 | .takes_value(true) 42 | .display_order(3) 43 | .help("The amount of concurrent requests"), 44 | ) 45 | .arg( 46 | Arg::with_name("workers") 47 | .short('w') 48 | .long("workers") 49 | .default_value("1") 50 | .takes_value(true) 51 | .display_order(5) 52 | .help("The amount of workers"), 53 | ) 54 | .get_matches(); 55 | 56 | let rate = match matches.value_of("rate").unwrap().parse::() { 57 | Ok(n) => n, 58 | Err(_) => { 59 | println!("{}", "could not parse rate, using default of 1000"); 60 | 1000 61 | } 62 | }; 63 | 64 | let concurrency = match matches.value_of("concurrency").unwrap().parse::() { 65 | Ok(n) => n, 66 | Err(_) => { 67 | println!("{}", "could not parse concurrency, using default of 100"); 68 | 100 69 | } 70 | }; 71 | 72 | let w: usize = match matches.value_of("workers").unwrap().parse::() { 73 | Ok(w) => w, 74 | Err(_) => { 75 | println!("{}", "could not parse workers, using default of 1"); 76 | 1 77 | } 78 | }; 79 | 80 | // Set up a worker pool with the number of threads specified from the arguments 81 | let rt = Builder::new_multi_thread() 82 | .enable_all() 83 | .worker_threads(w) 84 | .build() 85 | .unwrap(); 86 | 87 | // job channels 88 | let (job_tx, job_rx) = spmc::channel::(); 89 | 90 | rt.spawn(async move { send_url(job_tx, rate).await }); 91 | 92 | // process the jobs 93 | let workers = FuturesUnordered::new(); 94 | 95 | // process the jobs for scanning. 96 | for _ in 0..concurrency { 97 | let jrx = job_rx.clone(); 98 | workers.push(task::spawn(async move { 99 | // run the detector 100 | run_parser(jrx).await 101 | })); 102 | } 103 | let _: Vec<_> = workers.collect().await; 104 | rt.shutdown_background(); 105 | 106 | Ok(()) 107 | } 108 | 109 | async fn send_url( 110 | mut tx: spmc::Sender, 111 | rate: u32, 112 | ) -> Result<(), Box> { 113 | //set rate limit 114 | let lim = RateLimiter::direct(Quota::per_second(std::num::NonZeroU32::new(rate).unwrap())); 115 | 116 | let stdin = io::BufReader::new(io::stdin()); 117 | let mut lines = stdin.lines(); 118 | 119 | // send the jobs 120 | while let Some(line) = lines.next().await { 121 | let host = line.unwrap(); 122 | lim.until_ready().await; 123 | let msg = Job { 124 | host: Some(host.to_string().clone()), 125 | }; 126 | if let Err(_) = tx.send(msg) { 127 | continue; 128 | } 129 | } 130 | Ok(()) 131 | } 132 | 133 | pub async fn run_parser(rx: spmc::Receiver) { 134 | while let Ok(job) = rx.recv() { 135 | let job_host = job.host.unwrap(); 136 | let ext: TldExtractor = TldOption::default().build(); 137 | let extractor = match ext.extract(&job_host) { 138 | Ok(extractor) => extractor, 139 | Err(_) => continue, 140 | }; 141 | 142 | let mut root_domain = String::from(""); 143 | 144 | let domain = match extractor.domain { 145 | Some(domain) => domain, 146 | None => continue, 147 | }; 148 | 149 | let suffix = match extractor.suffix { 150 | Some(suffix) => suffix, 151 | None => continue, 152 | }; 153 | 154 | root_domain.push_str(&domain); 155 | root_domain.push_str("."); 156 | root_domain.push_str(&suffix); 157 | 158 | println!("{}", root_domain.to_string()); 159 | } 160 | } 161 | --------------------------------------------------------------------------------