├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── go.mod ├── main.go └── script └── release /.gitignore: -------------------------------------------------------------------------------- 1 | domains 2 | hosts 3 | httprobe 4 | *.tgz 5 | *.zip 6 | *.exe 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.11.4-alpine3.7 AS build-env 2 | RUN apk add --no-cache --upgrade git openssh-client ca-certificates 3 | RUN go get -u github.com/golang/dep/cmd/dep 4 | WORKDIR /go/src/app 5 | 6 | COPY . /go/src/app 7 | 8 | RUN go build -o httprobe main.go 9 | 10 | 11 | FROM alpine:3.9 12 | 13 | RUN apk add shadow bash && \ 14 | useradd --create-home --shell /sbin/nologin httprobe && \ 15 | mkdir /httprobe && \ 16 | chown httprobe:httprobe /httprobe 17 | 18 | COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 19 | COPY --from=build-env /go/src/app/httprobe /httprobe/httprobe 20 | 21 | 22 | USER httprobe 23 | WORKDIR /httprobe 24 | 25 | ENTRYPOINT ["/httprobe/httprobe"] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Tom Hudson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # httprobe 2 | 3 | Take a list of domains and probe for working http and https servers. 4 | 5 | ## Install 6 | 7 | ``` 8 | ▶ go install github.com/tomnomnom/httprobe@latest 9 | ``` 10 | 11 | ## Basic Usage 12 | 13 | httprobe accepts line-delimited domains on `stdin`: 14 | 15 | ``` 16 | ▶ cat recon/example/domains.txt 17 | example.com 18 | example.edu 19 | example.net 20 | ▶ cat recon/example/domains.txt | httprobe 21 | http://example.com 22 | http://example.net 23 | http://example.edu 24 | https://example.com 25 | https://example.edu 26 | https://example.net 27 | ``` 28 | 29 | ## Extra Probes 30 | 31 | By default httprobe checks for HTTP on port 80 and HTTPS on port 443. You can add additional 32 | probes with the `-p` flag by specifying a protocol and port pair: 33 | 34 | ``` 35 | ▶ cat domains.txt | httprobe -p http:81 -p https:8443 36 | ``` 37 | 38 | ## Concurrency 39 | 40 | You can set the concurrency level with the `-c` flag: 41 | 42 | ``` 43 | ▶ cat domains.txt | httprobe -c 50 44 | ``` 45 | 46 | ## Timeout 47 | 48 | You can change the timeout by using the `-t` flag and specifying a timeout in milliseconds: 49 | 50 | ``` 51 | ▶ cat domains.txt | httprobe -t 20000 52 | ``` 53 | 54 | ## Skipping Default Probes 55 | 56 | If you don't want to probe for HTTP on port 80 or HTTPS on port 443, you can use the 57 | `-s` flag. You'll need to specify the probes you do want using the `-p` flag: 58 | 59 | ``` 60 | ▶ cat domains.txt | httprobe -s -p https:8443 61 | ``` 62 | 63 | ## Prefer HTTPS 64 | 65 | Sometimes you don't care about checking HTTP if HTTPS is working. You can do that with the `--prefer-https` flag: 66 | 67 | ``` 68 | ▶ cat domains.txt | httprobe --prefer-https 69 | ``` 70 | 71 | ## Docker 72 | 73 | Build the docker container: 74 | 75 | ``` 76 | ▶ docker build -t httprobe . 77 | ``` 78 | 79 | Run the container, passing the contents of a file into stdin of the process inside the container. `-i` is required to correctly map `stdin` into the container and to the `httprobe` binary. 80 | 81 | ``` 82 | ▶ cat domains.txt | docker run -i httprobe 83 | ``` 84 | 85 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tomnomnom/httprobe 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "crypto/tls" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net" 11 | "net/http" 12 | "os" 13 | "strings" 14 | "sync" 15 | "time" 16 | ) 17 | 18 | type probeArgs []string 19 | 20 | func (p *probeArgs) Set(val string) error { 21 | *p = append(*p, val) 22 | return nil 23 | } 24 | 25 | func (p probeArgs) String() string { 26 | return strings.Join(p, ",") 27 | } 28 | 29 | func main() { 30 | 31 | // concurrency flag 32 | var concurrency int 33 | flag.IntVar(&concurrency, "c", 20, "set the concurrency level (split equally between HTTPS and HTTP requests)") 34 | 35 | // probe flags 36 | var probes probeArgs 37 | flag.Var(&probes, "p", "add additional probe (proto:port)") 38 | 39 | // skip default probes flag 40 | var skipDefault bool 41 | flag.BoolVar(&skipDefault, "s", false, "skip the default probes (http:80 and https:443)") 42 | 43 | // timeout flag 44 | var to int 45 | flag.IntVar(&to, "t", 10000, "timeout (milliseconds)") 46 | 47 | // prefer https 48 | var preferHTTPS bool 49 | flag.BoolVar(&preferHTTPS, "prefer-https", false, "only try plain HTTP if HTTPS fails") 50 | 51 | // HTTP method to use 52 | var method string 53 | flag.StringVar(&method, "method", "GET", "HTTP method to use") 54 | 55 | flag.Parse() 56 | 57 | // make an actual time.Duration out of the timeout 58 | timeout := time.Duration(to * 1000000) 59 | 60 | var tr = &http.Transport{ 61 | MaxIdleConns: 30, 62 | IdleConnTimeout: time.Second, 63 | DisableKeepAlives: true, 64 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 65 | DialContext: (&net.Dialer{ 66 | Timeout: timeout, 67 | KeepAlive: time.Second, 68 | }).DialContext, 69 | } 70 | 71 | re := func(req *http.Request, via []*http.Request) error { 72 | return http.ErrUseLastResponse 73 | } 74 | 75 | client := &http.Client{ 76 | Transport: tr, 77 | CheckRedirect: re, 78 | Timeout: timeout, 79 | } 80 | 81 | // domain/port pairs are initially sent on the httpsURLs channel. 82 | // If they are listening and the --prefer-https flag is set then 83 | // no HTTP check is performed; otherwise they're put onto the httpURLs 84 | // channel for an HTTP check. 85 | httpsURLs := make(chan string) 86 | httpURLs := make(chan string) 87 | output := make(chan string) 88 | 89 | // HTTPS workers 90 | var httpsWG sync.WaitGroup 91 | for i := 0; i < concurrency/2; i++ { 92 | httpsWG.Add(1) 93 | 94 | go func() { 95 | for url := range httpsURLs { 96 | 97 | // always try HTTPS first 98 | withProto := "https://" + url 99 | if isListening(client, withProto, method) { 100 | output <- withProto 101 | 102 | // skip trying HTTP if --prefer-https is set 103 | if preferHTTPS { 104 | continue 105 | } 106 | } 107 | 108 | httpURLs <- url 109 | } 110 | 111 | httpsWG.Done() 112 | }() 113 | } 114 | 115 | // HTTP workers 116 | var httpWG sync.WaitGroup 117 | for i := 0; i < concurrency/2; i++ { 118 | httpWG.Add(1) 119 | 120 | go func() { 121 | for url := range httpURLs { 122 | withProto := "http://" + url 123 | if isListening(client, withProto, method) { 124 | output <- withProto 125 | continue 126 | } 127 | } 128 | 129 | httpWG.Done() 130 | }() 131 | } 132 | 133 | // Close the httpURLs channel when the HTTPS workers are done 134 | go func() { 135 | httpsWG.Wait() 136 | close(httpURLs) 137 | }() 138 | 139 | // Output worker 140 | var outputWG sync.WaitGroup 141 | outputWG.Add(1) 142 | go func() { 143 | for o := range output { 144 | fmt.Println(o) 145 | } 146 | outputWG.Done() 147 | }() 148 | 149 | // Close the output channel when the HTTP workers are done 150 | go func() { 151 | httpWG.Wait() 152 | close(output) 153 | }() 154 | 155 | // accept domains on stdin 156 | sc := bufio.NewScanner(os.Stdin) 157 | for sc.Scan() { 158 | domain := strings.ToLower(sc.Text()) 159 | 160 | // submit standard port checks 161 | if !skipDefault { 162 | httpsURLs <- domain 163 | } 164 | 165 | // Adding port templates 166 | xlarge := []string{"81", "300", "591", "593", "832", "981", "1010", "1311", "2082", "2087", "2095", "2096", "2480", "3000", "3128", "3333", "4243", "4567", "4711", "4712", "4993", "5000", "5104", "5108", "5800", "6543", "7000", "7396", "7474", "8000", "8001", "8008", "8014", "8042", "8069", "8080", "8081", "8088", "8090", "8091", "8118", "8123", "8172", "8222", "8243", "8280", "8281", "8333", "8443", "8500", "8834", "8880", "8888", "8983", "9000", "9043", "9060", "9080", "9090", "9091", "9200", "9443", "9800", "9981", "12443", "16080", "18091", "18092", "20720", "28017"} 167 | large := []string{"81", "591", "2082", "2087", "2095", "2096", "3000", "8000", "8001", "8008", "8080", "8083", "8443", "8834", "8888"} 168 | 169 | // submit any additional proto:port probes 170 | for _, p := range probes { 171 | switch p { 172 | case "xlarge": 173 | for _, port := range xlarge { 174 | httpsURLs <- fmt.Sprintf("%s:%s", domain, port) 175 | } 176 | case "large": 177 | for _, port := range large { 178 | httpsURLs <- fmt.Sprintf("%s:%s", domain, port) 179 | } 180 | default: 181 | pair := strings.SplitN(p, ":", 2) 182 | if len(pair) != 2 { 183 | continue 184 | } 185 | 186 | // This is a little bit funny as "https" will imply an 187 | // http check as well unless the --prefer-https flag is 188 | // set. On balance I don't think that's *such* a bad thing 189 | // but it is maybe a little unexpected. 190 | if strings.ToLower(pair[0]) == "https" { 191 | httpsURLs <- fmt.Sprintf("%s:%s", domain, pair[1]) 192 | } else { 193 | httpURLs <- fmt.Sprintf("%s:%s", domain, pair[1]) 194 | } 195 | } 196 | } 197 | } 198 | 199 | // once we've sent all the URLs off we can close the 200 | // input/httpsURLs channel. The workers will finish what they're 201 | // doing and then call 'Done' on the WaitGroup 202 | close(httpsURLs) 203 | 204 | // check there were no errors reading stdin (unlikely) 205 | if err := sc.Err(); err != nil { 206 | fmt.Fprintf(os.Stderr, "failed to read input: %s\n", err) 207 | } 208 | 209 | // Wait until the output waitgroup is done 210 | outputWG.Wait() 211 | } 212 | 213 | func isListening(client *http.Client, url, method string) bool { 214 | 215 | req, err := http.NewRequest(method, url, nil) 216 | if err != nil { 217 | return false 218 | } 219 | 220 | req.Header.Add("Connection", "close") 221 | req.Close = true 222 | 223 | resp, err := client.Do(req) 224 | if resp != nil { 225 | io.Copy(ioutil.Discard, resp.Body) 226 | resp.Body.Close() 227 | } 228 | 229 | if err != nil { 230 | return false 231 | } 232 | 233 | return true 234 | } 235 | -------------------------------------------------------------------------------- /script/release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PROJDIR=$(cd `dirname $0`/.. && pwd) 3 | 4 | VERSION="${1}" 5 | TAG="v${VERSION}" 6 | USER="tomnomnom" 7 | REPO="httprobe" 8 | BINARY="${REPO}" 9 | 10 | if [[ -z "${VERSION}" ]]; then 11 | echo "Usage: ${0} " 12 | exit 1 13 | fi 14 | 15 | if [[ -z "${GITHUB_TOKEN}" ]]; then 16 | echo "You forgot to set your GITHUB_TOKEN" 17 | exit 2 18 | fi 19 | 20 | cd ${PROJDIR} 21 | 22 | # Run the tests 23 | go test 24 | if [ $? -ne 0 ]; then 25 | echo "Tests failed. Aborting." 26 | exit 3 27 | fi 28 | 29 | FILELIST="" 30 | 31 | for ARCH in "amd64" "386"; do 32 | for OS in "darwin" "linux" "windows" "freebsd"; do 33 | 34 | if [[ "${OS}" == "darwin" && "${ARCH}" == "386" ]]; then 35 | continue 36 | fi 37 | 38 | BINFILE="${BINARY}" 39 | 40 | if [[ "${OS}" == "windows" ]]; then 41 | BINFILE="${BINFILE}.exe" 42 | fi 43 | 44 | rm -f ${BINFILE} 45 | 46 | GOOS=${OS} GOARCH=${ARCH} go build github.com/${USER}/${REPO} 47 | 48 | if [[ "${OS}" == "windows" ]]; then 49 | ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.zip" 50 | zip ${ARCHIVE} ${BINFILE} 51 | rm ${BINFILE} 52 | else 53 | ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.tgz" 54 | tar --create --gzip --file=${ARCHIVE} ${BINFILE} 55 | fi 56 | 57 | FILELIST="${FILELIST} ${PROJDIR}/${ARCHIVE}" 58 | done 59 | done 60 | 61 | gh release create ${TAG} ${FILELIST} 62 | rm ${FILELIST} 63 | 64 | --------------------------------------------------------------------------------