├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ ├── feature_request.md │ └── issue-report.md ├── dependabot.yml ├── release.yml └── workflows │ ├── build-test.yml │ ├── codeql-analysis.yml │ ├── compat-checks.yaml │ ├── dep-auto-merge.yml │ ├── dockerhub-push.yml │ ├── release-binary.yml │ └── release-test.yml ├── .gitignore ├── DISCLAIMER.md ├── Dockerfile ├── LICENSE.md ├── README.md ├── THANKS.md ├── static ├── subfinder-logo.png └── subfinder-run.png └── v2 ├── .goreleaser.yml ├── Makefile ├── cmd └── subfinder │ └── main.go ├── examples └── main.go ├── go.mod ├── go.sum └── pkg ├── passive ├── doc.go ├── passive.go ├── sources.go ├── sources_test.go ├── sources_w_auth_test.go └── sources_wo_auth_test.go ├── resolve ├── client.go ├── doc.go └── resolve.go ├── runner ├── banners.go ├── config.go ├── doc.go ├── enumerate.go ├── enumerate_test.go ├── initialize.go ├── options.go ├── outputter.go ├── runner.go ├── stats.go ├── util.go └── validate.go ├── subscraping ├── agent.go ├── doc.go ├── extractor.go ├── sources │ ├── alienvault │ │ └── alienvault.go │ ├── anubis │ │ └── anubis.go │ ├── bevigil │ │ └── bevigil.go │ ├── binaryedge │ │ └── binaryedge.go │ ├── bufferover │ │ └── bufferover.go │ ├── builtwith │ │ └── builtwith.go │ ├── c99 │ │ └── c99.go │ ├── censys │ │ └── censys.go │ ├── certspotter │ │ └── certspotter.go │ ├── chaos │ │ └── chaos.go │ ├── chinaz │ │ └── chinaz.go │ ├── commoncrawl │ │ └── commoncrawl.go │ ├── crtsh │ │ └── crtsh.go │ ├── digitalyama │ │ └── digitalyama.go │ ├── digitorus │ │ └── digitorus.go │ ├── dnsdb │ │ └── dnsdb.go │ ├── dnsdumpster │ │ └── dnsdumpster.go │ ├── dnsrepo │ │ └── dnsrepo.go │ ├── facebook │ │ ├── ctlogs.go │ │ ├── ctlogs_test.go │ │ └── types.go │ ├── fofa │ │ └── fofa.go │ ├── fullhunt │ │ └── fullhunt.go │ ├── github │ │ ├── github.go │ │ └── tokenmanager.go │ ├── gitlab │ │ └── gitlab.go │ ├── hackertarget │ │ └── hackertarget.go │ ├── hudsonrock │ │ └── hudsonrock.go │ ├── hunter │ │ └── hunter.go │ ├── intelx │ │ └── intelx.go │ ├── leakix │ │ └── leakix.go │ ├── netlas │ │ └── netlas.go │ ├── quake │ │ └── quake.go │ ├── rapiddns │ │ └── rapiddns.go │ ├── reconcloud │ │ └── reconcloud.go │ ├── redhuntlabs │ │ └── redhuntlabs.go │ ├── riddler │ │ └── riddler.go │ ├── robtex │ │ └── robtext.go │ ├── securitytrails │ │ └── securitytrails.go │ ├── shodan │ │ └── shodan.go │ ├── sitedossier │ │ └── sitedossier.go │ ├── threatbook │ │ └── threatbook.go │ ├── threatcrowd │ │ └── threatcrowd.go │ ├── threatminer │ │ └── threatminer.go │ ├── virustotal │ │ └── virustotal.go │ ├── waybackarchive │ │ └── waybackarchive.go │ ├── whoisxmlapi │ │ └── whoisxmlapi.go │ └── zoomeyeapi │ │ └── zoomeyeapi.go ├── types.go └── utils.go └── testutils └── integration.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[Issue] " 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Subfinder version** 14 | Include the version of subfinder you are using, `subfinder -version` 15 | 16 | **Complete command you used to reproduce this** 17 | 18 | 19 | **Screenshots** 20 | Add screenshots of the error for a better context. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | 3 | contact_links: 4 | - name: Ask an question / advise on using subfinder 5 | url: https://github.com/projectdiscovery/subfinder/discussions/categories/q-a 6 | about: Ask a question or request support for using subfinder 7 | 8 | - name: Share idea / feature to discuss for subfinder 9 | url: https://github.com/projectdiscovery/subfinder/discussions/categories/ideas 10 | about: Share idea / feature to discuss for subfinder 11 | 12 | - name: Connect with PD Team (Discord) 13 | url: https://discord.gg/projectdiscovery 14 | about: Connect with PD Team for direct communication -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Request feature to implement in this project 4 | labels: 'Type: Enhancement' 5 | --- 6 | 7 | 13 | 14 | ### Please describe your feature request: 15 | 16 | 17 | ### Describe the use case of this feature: 18 | 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue report 3 | about: Create a report to help us to improve the project 4 | labels: 'Type: Bug' 5 | 6 | --- 7 | 8 | 13 | 14 | 15 | 16 | ### Subfinder version: 17 | 18 | 19 | 20 | 21 | ### Current Behavior: 22 | 23 | 24 | ### Expected Behavior: 25 | 26 | 27 | ### Steps To Reproduce: 28 | 33 | 34 | 35 | ### Anything else: 36 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | 9 | # Maintain dependencies for go modules 10 | - package-ecosystem: "gomod" 11 | directory: "v2/" 12 | schedule: 13 | interval: "weekly" 14 | target-branch: "dev" 15 | commit-message: 16 | prefix: "chore" 17 | include: "scope" 18 | allow: 19 | - dependency-name: "github.com/projectdiscovery/*" 20 | groups: 21 | modules: 22 | patterns: ["github.com/projectdiscovery/*"] 23 | labels: 24 | - "Type: Maintenance" 25 | 26 | # # Maintain dependencies for GitHub Actions 27 | # - package-ecosystem: "github-actions" 28 | # directory: "/" 29 | # schedule: 30 | # interval: "weekly" 31 | # target-branch: "dev" 32 | # commit-message: 33 | # prefix: "chore" 34 | # include: "scope" 35 | # labels: 36 | # - "Type: Maintenance" 37 | # 38 | # # Maintain dependencies for docker 39 | # - package-ecosystem: "docker" 40 | # directory: "/" 41 | # schedule: 42 | # interval: "weekly" 43 | # target-branch: "dev" 44 | # commit-message: 45 | # prefix: "chore" 46 | # include: "scope" 47 | # labels: 48 | # - "Type: Maintenance" -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | authors: 4 | - dependabot 5 | categories: 6 | - title: 🎉 New Features 7 | labels: 8 | - "Type: Enhancement" 9 | - title: 🐞 Bug Fixes 10 | labels: 11 | - "Type: Bug" 12 | - title: 🔨 Maintenance 13 | labels: 14 | - "Type: Maintenance" 15 | - title: Other Changes 16 | labels: 17 | - "*" -------------------------------------------------------------------------------- /.github/workflows/build-test.yml: -------------------------------------------------------------------------------- 1 | name: 🔨 Build Test 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '**.go' 7 | - '**.mod' 8 | workflow_dispatch: 9 | inputs: 10 | short: 11 | description: 'Use -short flag for tests' 12 | required: false 13 | type: boolean 14 | default: false 15 | 16 | jobs: 17 | lint: 18 | name: Lint Test 19 | if: "${{ !endsWith(github.actor, '[bot]') }}" 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v4 23 | - uses: projectdiscovery/actions/setup/go@v1 24 | - name: Run golangci-lint 25 | uses: projectdiscovery/actions/golangci-lint/v2@v1 26 | with: 27 | version: latest 28 | args: --timeout 5m 29 | working-directory: v2/ 30 | 31 | build: 32 | name: Test Builds 33 | needs: [lint] 34 | runs-on: ${{ matrix.os }} 35 | strategy: 36 | matrix: 37 | os: [ubuntu-latest, windows-latest, macOS-13] 38 | steps: 39 | - uses: actions/checkout@v4 40 | - uses: projectdiscovery/actions/setup/go@v1 41 | - run: go build ./... 42 | working-directory: v2/ 43 | 44 | - name: Run tests 45 | env: 46 | BEVIGIL_API_KEY: ${{secrets.BEVIGIL_API_KEY}} 47 | BINARYEDGE_API_KEY: ${{secrets.BINARYEDGE_API_KEY}} 48 | BUFFEROVER_API_KEY: ${{secrets.BUFFEROVER_API_KEY}} 49 | C99_API_KEY: ${{secrets.C99_API_KEY}} 50 | CENSYS_API_KEY: ${{secrets.CENSYS_API_KEY}} 51 | CERTSPOTTER_API_KEY: ${{secrets.CERTSPOTTER_API_KEY}} 52 | CHAOS_API_KEY: ${{secrets.CHAOS_API_KEY}} 53 | CHINAZ_API_KEY: ${{secrets.CHINAZ_API_KEY}} 54 | DNSDB_API_KEY: ${{secrets.DNSDB_API_KEY}} 55 | DNSREPO_API_KEY: ${{secrets.DNSREPO_API_KEY}} 56 | FOFA_API_KEY: ${{secrets.FOFA_API_KEY}} 57 | FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}} 58 | GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}} 59 | HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}} 60 | INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} 61 | LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} 62 | QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} 63 | ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}} 64 | SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} 65 | SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} 66 | THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} 67 | VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} 68 | WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} 69 | ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} 70 | uses: nick-invision/retry@v2 71 | with: 72 | timeout_seconds: 360 73 | max_attempts: 3 74 | command: cd v2; go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} 75 | 76 | - name: Race Condition Tests 77 | run: go build -race ./... 78 | working-directory: v2/ 79 | 80 | - name: Run Example 81 | run: go run . 82 | working-directory: v2/examples -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: 🚨 CodeQL Analysis 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | paths: 7 | - '**.go' 8 | - '**.mod' 9 | 10 | jobs: 11 | analyze: 12 | name: Analyze 13 | runs-on: ubuntu-latest 14 | permissions: 15 | actions: read 16 | contents: read 17 | security-events: write 18 | 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | language: [ 'go' ] 23 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 24 | 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@v3 28 | 29 | # Initializes the CodeQL tools for scanning. 30 | - name: Initialize CodeQL 31 | uses: github/codeql-action/init@v2 32 | with: 33 | languages: ${{ matrix.language }} 34 | 35 | - name: Autobuild 36 | uses: github/codeql-action/autobuild@v2 37 | 38 | - name: Perform CodeQL Analysis 39 | uses: github/codeql-action/analyze@v2 -------------------------------------------------------------------------------- /.github/workflows/compat-checks.yaml: -------------------------------------------------------------------------------- 1 | name: ♾️ Compatibility Checks 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize] 6 | branches: 7 | - dev 8 | 9 | jobs: 10 | check: 11 | if: github.actor == 'dependabot[bot]' 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: write 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: projectdiscovery/actions/setup/go/compat-checks@master 18 | with: 19 | go-version-file: 'v2/go.mod' 20 | 21 | -------------------------------------------------------------------------------- /.github/workflows/dep-auto-merge.yml: -------------------------------------------------------------------------------- 1 | name: 🤖 dep auto merge 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - dev 7 | workflow_dispatch: 8 | 9 | permissions: 10 | pull-requests: write 11 | issues: write 12 | repository-projects: write 13 | 14 | jobs: 15 | automerge: 16 | runs-on: ubuntu-latest 17 | if: github.actor == 'dependabot[bot]' 18 | steps: 19 | - uses: actions/checkout@v3 20 | with: 21 | token: ${{ secrets.DEPENDABOT_PAT }} 22 | 23 | - uses: ahmadnassri/action-dependabot-auto-merge@v2 24 | with: 25 | github-token: ${{ secrets.DEPENDABOT_PAT }} 26 | target: all -------------------------------------------------------------------------------- /.github/workflows/dockerhub-push.yml: -------------------------------------------------------------------------------- 1 | name: 🌥 Docker Push 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["🎉 Release Binary"] 6 | types: 7 | - completed 8 | workflow_dispatch: 9 | 10 | jobs: 11 | docker: 12 | runs-on: ubuntu-latest-16-cores 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v3 16 | 17 | - name: Get Github tag 18 | id: meta 19 | run: | 20 | curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT 21 | 22 | - name: Set up QEMU 23 | uses: docker/setup-qemu-action@v2 24 | 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v2 27 | 28 | - name: Login to DockerHub 29 | uses: docker/login-action@v2 30 | with: 31 | username: ${{ secrets.DOCKER_USERNAME }} 32 | password: ${{ secrets.DOCKER_TOKEN }} 33 | 34 | - name: Build and push 35 | uses: docker/build-push-action@v4 36 | with: 37 | context: . 38 | platforms: linux/amd64,linux/arm64,linux/arm 39 | push: true 40 | tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} -------------------------------------------------------------------------------- /.github/workflows/release-binary.yml: -------------------------------------------------------------------------------- 1 | name: 🎉 Release Binary 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | workflow_dispatch: 8 | 9 | jobs: 10 | release: 11 | runs-on: ubuntu-latest-16-cores 12 | steps: 13 | - name: "Check out code" 14 | uses: actions/checkout@v3 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: "Set up Go" 19 | uses: actions/setup-go@v4 20 | with: 21 | go-version: 1.21.x 22 | 23 | - name: "Create release on GitHub" 24 | uses: goreleaser/goreleaser-action@v3 25 | with: 26 | args: "release --clean" 27 | version: latest 28 | workdir: v2/ 29 | env: 30 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 31 | SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" 32 | DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" 33 | DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" -------------------------------------------------------------------------------- /.github/workflows/release-test.yml: -------------------------------------------------------------------------------- 1 | name: 🔨 Release Test 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - '**.go' 7 | - '**.mod' 8 | workflow_dispatch: 9 | 10 | jobs: 11 | release-test: 12 | runs-on: ubuntu-latest-16-cores 13 | steps: 14 | - name: "Check out code" 15 | uses: actions/checkout@v3 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v4 21 | with: 22 | go-version: 1.21.x 23 | 24 | - name: release test 25 | uses: goreleaser/goreleaser-action@v4 26 | with: 27 | args: "release --clean --snapshot" 28 | version: latest 29 | workdir: v2 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | cmd/subfinder/subfinder 3 | # subfinder binary when built with `go build` 4 | v2/cmd/subfinder/subfinder 5 | # subfinder binary when built with `make` 6 | v2/subfinder 7 | vendor/ 8 | .idea 9 | .devcontainer 10 | .vscode 11 | dist -------------------------------------------------------------------------------- /DISCLAIMER.md: -------------------------------------------------------------------------------- 1 | ## Disclaimer 2 | 3 | Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services: 4 | 5 | - Bufferover: https://tls.bufferover.run 6 | - CommonCrawl: https://commoncrawl.org/terms-of-use/full 7 | - certspotter: https://sslmate.com/terms 8 | - dnsdumpster: https://hackertarget.com/terms 9 | - Google Transparency: https://policies.google.com/terms 10 | - Alienvault: https://www.alienvault.com/terms/website-terms-of-use07may2018 11 | 12 | --- 13 | 14 | You expressly understand and agree that Subfinder (creators and contributors) shall not be liable for any damages or losses resulting from your use of this tool or third-party products that use it. 15 | 16 | Creators aren't in charge of any and have/has no responsibility for any kind of: 17 | 18 | - Unlawful or illegal use of the tool. 19 | - Legal or Law infringement (acted in any country, state, municipality, place) by third parties and users. 20 | - Act against ethical and / or human moral, ethic, and peoples and cultures of the world. 21 | - Malicious act, capable of causing damage to third parties, promoted or distributed by third parties or the user through this tool. 22 | 23 | 24 | ### Contact 25 | 26 | Please contact at contact@projectdiscovery.io for any questions. 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build 2 | FROM golang:1.24-alpine AS build-env 3 | RUN apk add build-base 4 | WORKDIR /app 5 | COPY . /app 6 | WORKDIR /app/v2 7 | RUN go mod download 8 | RUN go build ./cmd/subfinder 9 | 10 | # Release 11 | FROM alpine:latest 12 | RUN apk upgrade --no-cache \ 13 | && apk add --no-cache bind-tools ca-certificates 14 | COPY --from=build-env /app/v2/subfinder /usr/local/bin/ 15 | 16 | ENTRYPOINT ["subfinder"] 17 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ProjectDiscovery, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /THANKS.md: -------------------------------------------------------------------------------- 1 | ### Thanks 2 | 3 | Many people have contributed to subfinder making it a wonderful tool either by making a pull request fixing some stuff or giving generous donations to support the further development of this tool. Here, we recognize these persons and thank them. 4 | 5 | - All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/subfinder/graphs/contributors) who made subfinder what it is. 6 | 7 | We'd like to thank some additional amazing people, who contributed a lot in subfinder's journey - 8 | 9 | - [@vzamanillo](https://github.com/vzamanillo) - For adding multiple features and overall project improvements. 10 | - [@infosec-au](https://github.com/infosec-au) - Donating to the project. 11 | - [@codingo](https://github.com/codingo) - Initial work on the project, managing it, lot of work! 12 | - [@picatz](https://github.com/picatz) - Improving the structure of the project a lot. New ideas! -------------------------------------------------------------------------------- /static/subfinder-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/projectdiscovery/subfinder/98e678907ac49a1036dd6f60fa6d318821d3d115/static/subfinder-logo.png -------------------------------------------------------------------------------- /static/subfinder-run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/projectdiscovery/subfinder/98e678907ac49a1036dd6f60fa6d318821d3d115/static/subfinder-run.png -------------------------------------------------------------------------------- /v2/.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod tidy 4 | 5 | builds: 6 | - env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - windows 10 | - linux 11 | - darwin 12 | goarch: 13 | - amd64 14 | - 386 15 | - arm 16 | - arm64 17 | 18 | ignore: 19 | - goos: darwin 20 | goarch: '386' 21 | - goos: windows 22 | goarch: 'arm' 23 | 24 | binary: '{{ .ProjectName }}' 25 | main: cmd/subfinder/main.go 26 | 27 | archives: 28 | - format: zip 29 | name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' 30 | 31 | checksum: 32 | algorithm: sha256 33 | 34 | announce: 35 | slack: 36 | enabled: true 37 | channel: '#release' 38 | username: GoReleaser 39 | message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' 40 | 41 | discord: 42 | enabled: true 43 | message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' -------------------------------------------------------------------------------- /v2/Makefile: -------------------------------------------------------------------------------- 1 | # Go parameters 2 | GOCMD=go 3 | GOBUILD=$(GOCMD) build 4 | GOMOD=$(GOCMD) mod 5 | GOTEST=$(GOCMD) test 6 | GOFLAGS := -v 7 | LDFLAGS := -s -w 8 | 9 | ifneq ($(shell go env GOOS),darwin) 10 | LDFLAGS := -extldflags "-static" 11 | endif 12 | 13 | all: build 14 | build: 15 | $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "subfinder" cmd/subfinder/main.go 16 | test: 17 | $(GOTEST) $(GOFLAGS) ./... 18 | tidy: 19 | $(GOMOD) tidy 20 | -------------------------------------------------------------------------------- /v2/cmd/subfinder/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/projectdiscovery/subfinder/v2/pkg/runner" 5 | // Attempts to increase the OS file descriptors - Fail silently 6 | _ "github.com/projectdiscovery/fdmax/autofdmax" 7 | "github.com/projectdiscovery/gologger" 8 | ) 9 | 10 | func main() { 11 | // Parse the command line flags and read config files 12 | options := runner.ParseOptions() 13 | 14 | newRunner, err := runner.NewRunner(options) 15 | if err != nil { 16 | gologger.Fatal().Msgf("Could not create runner: %s\n", err) 17 | } 18 | 19 | err = newRunner.RunEnumeration() 20 | if err != nil { 21 | gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /v2/examples/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "log" 8 | 9 | "github.com/projectdiscovery/subfinder/v2/pkg/runner" 10 | ) 11 | 12 | func main() { 13 | subfinderOpts := &runner.Options{ 14 | Threads: 10, // Thread controls the number of threads to use for active enumerations 15 | Timeout: 30, // Timeout is the seconds to wait for sources to respond 16 | MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration 17 | // ResultCallback: func(s *resolve.HostEntry) { 18 | // callback function executed after each unique subdomain is found 19 | // }, 20 | // ProviderConfig: "your_provider_config.yaml", 21 | // and other config related options 22 | } 23 | 24 | // disable timestamps in logs / configure logger 25 | log.SetFlags(0) 26 | 27 | subfinder, err := runner.NewRunner(subfinderOpts) 28 | if err != nil { 29 | log.Fatalf("failed to create subfinder runner: %v", err) 30 | } 31 | 32 | output := &bytes.Buffer{} 33 | var sourceMap map[string]map[string]struct{} 34 | // To run subdomain enumeration on a single domain 35 | if sourceMap, err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil { 36 | log.Fatalf("failed to enumerate single domain: %v", err) 37 | } 38 | 39 | // To run subdomain enumeration on a list of domains from file/reader 40 | // file, err := os.Open("domains.txt") 41 | // if err != nil { 42 | // log.Fatalf("failed to open domains file: %v", err) 43 | // } 44 | // defer file.Close() 45 | // if err = subfinder.EnumerateMultipleDomainsWithCtx(context.Background(), file, []io.Writer{output}); err != nil { 46 | // log.Fatalf("failed to enumerate subdomains from file: %v", err) 47 | // } 48 | 49 | // print the output 50 | log.Println(output.String()) 51 | 52 | // Or use sourceMap to access the results in your application 53 | for subdomain, sources := range sourceMap { 54 | sourcesList := make([]string, 0, len(sources)) 55 | for source := range sources { 56 | sourcesList = append(sourcesList, source) 57 | } 58 | log.Printf("%s %s (%d)\n", subdomain, sourcesList, len(sources)) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /v2/pkg/passive/doc.go: -------------------------------------------------------------------------------- 1 | // Package passive provides capability for doing passive subdomain 2 | // enumeration on targets. 3 | package passive 4 | -------------------------------------------------------------------------------- /v2/pkg/passive/passive.go: -------------------------------------------------------------------------------- 1 | package passive 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "sort" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "github.com/projectdiscovery/ratelimit" 13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 14 | ) 15 | 16 | type EnumerationOptions struct { 17 | customRateLimiter *subscraping.CustomRateLimit 18 | } 19 | 20 | type EnumerateOption func(opts *EnumerationOptions) 21 | 22 | func WithCustomRateLimit(crl *subscraping.CustomRateLimit) EnumerateOption { 23 | return func(opts *EnumerationOptions) { 24 | opts.customRateLimiter = crl 25 | } 26 | } 27 | 28 | // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context 29 | func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result { 30 | return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime, options...) 31 | } 32 | 33 | // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain 34 | func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result { 35 | results := make(chan subscraping.Result) 36 | 37 | go func() { 38 | defer close(results) 39 | 40 | var enumerateOptions EnumerationOptions 41 | for _, enumerateOption := range options { 42 | enumerateOption(&enumerateOptions) 43 | } 44 | 45 | multiRateLimiter, err := a.buildMultiRateLimiter(ctx, rateLimit, enumerateOptions.customRateLimiter) 46 | if err != nil { 47 | results <- subscraping.Result{ 48 | Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err), 49 | } 50 | return 51 | } 52 | session, err := subscraping.NewSession(domain, proxy, multiRateLimiter, timeout) 53 | if err != nil { 54 | results <- subscraping.Result{ 55 | Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err), 56 | } 57 | return 58 | } 59 | defer session.Close() 60 | 61 | ctx, cancel := context.WithTimeout(ctx, maxEnumTime) 62 | 63 | wg := &sync.WaitGroup{} 64 | // Run each source in parallel on the target domain 65 | for _, runner := range a.sources { 66 | wg.Add(1) 67 | go func(source subscraping.Source) { 68 | ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name()) 69 | for resp := range source.Run(ctxWithValue, domain, session) { 70 | results <- resp 71 | } 72 | wg.Done() 73 | }(runner) 74 | } 75 | wg.Wait() 76 | cancel() 77 | }() 78 | return results 79 | } 80 | 81 | func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { 82 | var multiRateLimiter *ratelimit.MultiLimiter 83 | var err error 84 | for _, source := range a.sources { 85 | var rl uint 86 | if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { 87 | rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit) 88 | } 89 | 90 | if rl > 0 { 91 | multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second) 92 | } else { 93 | multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond) 94 | } 95 | 96 | if err != nil { 97 | break 98 | } 99 | } 100 | return multiRateLimiter, err 101 | } 102 | 103 | func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint { 104 | if sourceRateLimit > 0 { 105 | return sourceRateLimit 106 | } 107 | return defaultRateLimit 108 | } 109 | 110 | func addRateLimiter(ctx context.Context, multiRateLimiter *ratelimit.MultiLimiter, key string, maxCount uint, duration time.Duration) (*ratelimit.MultiLimiter, error) { 111 | if multiRateLimiter == nil { 112 | mrl, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ 113 | Key: key, 114 | IsUnlimited: maxCount == math.MaxUint32, 115 | MaxCount: maxCount, 116 | Duration: duration, 117 | }) 118 | return mrl, err 119 | } 120 | err := multiRateLimiter.Add(&ratelimit.Options{ 121 | Key: key, 122 | IsUnlimited: maxCount == math.MaxUint32, 123 | MaxCount: maxCount, 124 | Duration: duration, 125 | }) 126 | return multiRateLimiter, err 127 | } 128 | 129 | func (a *Agent) GetStatistics() map[string]subscraping.Statistics { 130 | stats := make(map[string]subscraping.Statistics) 131 | sort.Slice(a.sources, func(i, j int) bool { 132 | return a.sources[i].Name() > a.sources[j].Name() 133 | }) 134 | 135 | for _, source := range a.sources { 136 | stats[source.Name()] = source.Statistics() 137 | } 138 | return stats 139 | } 140 | -------------------------------------------------------------------------------- /v2/pkg/passive/sources_test.go: -------------------------------------------------------------------------------- 1 | package passive 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "golang.org/x/exp/maps" 10 | ) 11 | 12 | var ( 13 | expectedAllSources = []string{ 14 | "alienvault", 15 | "anubis", 16 | "bevigil", 17 | "binaryedge", 18 | "bufferover", 19 | "c99", 20 | "censys", 21 | "certspotter", 22 | "chaos", 23 | "chinaz", 24 | "commoncrawl", 25 | "crtsh", 26 | "digitorus", 27 | "dnsdumpster", 28 | "dnsdb", 29 | "dnsrepo", 30 | "fofa", 31 | "fullhunt", 32 | "github", 33 | "hackertarget", 34 | "intelx", 35 | "netlas", 36 | "quake", 37 | "rapiddns", 38 | "redhuntlabs", 39 | // "riddler", // failing due to cloudfront protection 40 | "robtex", 41 | "securitytrails", 42 | "shodan", 43 | "sitedossier", 44 | "threatbook", 45 | "threatcrowd", 46 | "virustotal", 47 | "waybackarchive", 48 | "whoisxmlapi", 49 | "zoomeyeapi", 50 | "hunter", 51 | "leakix", 52 | "facebook", 53 | // "threatminer", 54 | // "reconcloud", 55 | "builtwith", 56 | "hudsonrock", 57 | "digitalyama", 58 | } 59 | 60 | expectedDefaultSources = []string{ 61 | "alienvault", 62 | "anubis", 63 | "bevigil", 64 | "bufferover", 65 | "c99", 66 | "certspotter", 67 | "censys", 68 | "chaos", 69 | "chinaz", 70 | "crtsh", 71 | "digitorus", 72 | "dnsdumpster", 73 | "dnsrepo", 74 | "fofa", 75 | "fullhunt", 76 | "hackertarget", 77 | "intelx", 78 | "quake", 79 | "redhuntlabs", 80 | "robtex", 81 | // "riddler", // failing due to cloudfront protection 82 | "securitytrails", 83 | "shodan", 84 | "virustotal", 85 | "whoisxmlapi", 86 | "hunter", 87 | "leakix", 88 | "facebook", 89 | // "threatminer", 90 | // "reconcloud", 91 | "builtwith", 92 | "digitalyama", 93 | } 94 | 95 | expectedDefaultRecursiveSources = []string{ 96 | "alienvault", 97 | "binaryedge", 98 | "bufferover", 99 | "certspotter", 100 | "crtsh", 101 | "dnsdb", 102 | "digitorus", 103 | "hackertarget", 104 | "securitytrails", 105 | "virustotal", 106 | "leakix", 107 | "facebook", 108 | // "reconcloud", 109 | } 110 | ) 111 | 112 | func TestSourceCategorization(t *testing.T) { 113 | defaultSources := make([]string, 0, len(AllSources)) 114 | recursiveSources := make([]string, 0, len(AllSources)) 115 | for _, source := range AllSources { 116 | sourceName := source.Name() 117 | if source.IsDefault() { 118 | defaultSources = append(defaultSources, sourceName) 119 | } 120 | 121 | if source.HasRecursiveSupport() { 122 | recursiveSources = append(recursiveSources, sourceName) 123 | } 124 | } 125 | 126 | assert.ElementsMatch(t, expectedDefaultSources, defaultSources) 127 | assert.ElementsMatch(t, expectedDefaultRecursiveSources, recursiveSources) 128 | assert.ElementsMatch(t, expectedAllSources, maps.Keys(NameSourceMap)) 129 | } 130 | 131 | // Review: not sure if this test is necessary/useful 132 | // implementation is straightforward where sources are stored in maps and filtered based on options 133 | // the test is just checking if the filtering works as expected using count of sources 134 | func TestSourceFiltering(t *testing.T) { 135 | someSources := []string{ 136 | "alienvault", 137 | "chaos", 138 | "crtsh", 139 | "virustotal", 140 | } 141 | 142 | someExclusions := []string{ 143 | "alienvault", 144 | "virustotal", 145 | } 146 | 147 | tests := []struct { 148 | sources []string 149 | exclusions []string 150 | withAllSources bool 151 | withRecursion bool 152 | expectedLength int 153 | }{ 154 | {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, 155 | {someSources, someExclusions, false, true, 1}, 156 | {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, 157 | 158 | {someSources, []string{}, false, false, len(someSources)}, 159 | {someSources, []string{}, true, false, len(AllSources)}, 160 | 161 | {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, 162 | {[]string{}, []string{}, true, false, len(AllSources)}, 163 | {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, 164 | } 165 | for index, test := range tests { 166 | t.Run(strconv.Itoa(index+1), func(t *testing.T) { 167 | agent := New(test.sources, test.exclusions, test.withAllSources, test.withRecursion) 168 | 169 | for _, v := range agent.sources { 170 | fmt.Println(v.Name()) 171 | } 172 | 173 | assert.Equal(t, test.expectedLength, len(agent.sources)) 174 | agent = nil 175 | }) 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /v2/pkg/passive/sources_w_auth_test.go: -------------------------------------------------------------------------------- 1 | package passive 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "os" 8 | "reflect" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/assert" 14 | 15 | "github.com/projectdiscovery/gologger" 16 | "github.com/projectdiscovery/gologger/levels" 17 | "github.com/projectdiscovery/ratelimit" 18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 19 | ) 20 | 21 | func TestSourcesWithKeys(t *testing.T) { 22 | if testing.Short() { 23 | t.Skip("skipping test in short mode.") 24 | } 25 | 26 | domain := "hackerone.com" 27 | timeout := 60 28 | 29 | gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) 30 | 31 | ctxParent := context.Background() 32 | var multiRateLimiter *ratelimit.MultiLimiter 33 | for _, source := range AllSources { 34 | if !source.NeedsKey() { 35 | continue 36 | } 37 | multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond) 38 | } 39 | 40 | session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) 41 | assert.Nil(t, err) 42 | 43 | var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} 44 | 45 | for _, source := range AllSources { 46 | if !source.NeedsKey() { 47 | continue 48 | } 49 | 50 | var apiKey string 51 | if source.Name() == "chaos" { 52 | apiKey = os.Getenv("PDCP_API_KEY") 53 | } else { 54 | apiKey = os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))) 55 | } 56 | if apiKey == "" { 57 | fmt.Printf("Skipping %s as no API key is provided\n", source.Name()) 58 | continue 59 | } 60 | source.AddApiKeys([]string{apiKey}) 61 | 62 | t.Run(source.Name(), func(t *testing.T) { 63 | var results []subscraping.Result 64 | 65 | ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name()) 66 | for result := range source.Run(ctxWithValue, domain, session) { 67 | results = append(results, result) 68 | 69 | assert.Equal(t, source.Name(), result.Source, "wrong source name") 70 | 71 | if result.Type != subscraping.Error { 72 | assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)), 73 | fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value)) 74 | } else { 75 | assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error)) 76 | } 77 | } 78 | 79 | assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name())) 80 | }) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /v2/pkg/passive/sources_wo_auth_test.go: -------------------------------------------------------------------------------- 1 | package passive 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "reflect" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "golang.org/x/exp/slices" 14 | 15 | "github.com/projectdiscovery/gologger" 16 | "github.com/projectdiscovery/gologger/levels" 17 | "github.com/projectdiscovery/ratelimit" 18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 19 | ) 20 | 21 | func TestSourcesWithoutKeys(t *testing.T) { 22 | if testing.Short() { 23 | t.Skip("skipping test in short mode.") 24 | } 25 | 26 | ignoredSources := []string{ 27 | "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ 28 | "riddler", // failing due to cloudfront protection 29 | "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. 30 | "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally 31 | "waybackarchive", // Fails randomly 32 | "alienvault", // 503 Service Temporarily Unavailable 33 | "digitorus", // failing with "Failed to retrieve certificate" 34 | "dnsdumpster", // failing with "unexpected status code 403 received" 35 | } 36 | 37 | domain := "hackerone.com" 38 | timeout := 60 39 | 40 | gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) 41 | 42 | ctxParent := context.Background() 43 | 44 | var multiRateLimiter *ratelimit.MultiLimiter 45 | for _, source := range AllSources { 46 | if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) { 47 | continue 48 | } 49 | multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond) 50 | } 51 | 52 | session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) 53 | assert.Nil(t, err) 54 | 55 | var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} 56 | 57 | for _, source := range AllSources { 58 | if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) { 59 | continue 60 | } 61 | 62 | t.Run(source.Name(), func(t *testing.T) { 63 | var results []subscraping.Result 64 | 65 | ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name()) 66 | for result := range source.Run(ctxWithValue, domain, session) { 67 | results = append(results, result) 68 | 69 | assert.Equal(t, source.Name(), result.Source, "wrong source name") 70 | 71 | if result.Type != subscraping.Error { 72 | assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)), 73 | fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value)) 74 | } else { 75 | assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error)) 76 | } 77 | } 78 | 79 | assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name())) 80 | }) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /v2/pkg/resolve/client.go: -------------------------------------------------------------------------------- 1 | package resolve 2 | 3 | import ( 4 | "github.com/projectdiscovery/dnsx/libs/dnsx" 5 | ) 6 | 7 | // DefaultResolvers contains the default list of resolvers known to be good 8 | var DefaultResolvers = []string{ 9 | "1.1.1.1:53", // Cloudflare primary 10 | "1.0.0.1:53", // Cloudflare secondary 11 | "8.8.8.8:53", // Google primary 12 | "8.8.4.4:53", // Google secondary 13 | "9.9.9.9:53", // Quad9 Primary 14 | "9.9.9.10:53", // Quad9 Secondary 15 | "77.88.8.8:53", // Yandex Primary 16 | "77.88.8.1:53", // Yandex Secondary 17 | "208.67.222.222:53", // OpenDNS Primary 18 | "208.67.220.220:53", // OpenDNS Secondary 19 | } 20 | 21 | // Resolver is a struct for resolving DNS names 22 | type Resolver struct { 23 | DNSClient *dnsx.DNSX 24 | Resolvers []string 25 | } 26 | 27 | // New creates a new resolver struct with the default resolvers 28 | func New() *Resolver { 29 | return &Resolver{ 30 | Resolvers: []string{}, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /v2/pkg/resolve/doc.go: -------------------------------------------------------------------------------- 1 | // Package resolve is used to handle resolving records 2 | // It also handles wildcard subdomains and rotating resolvers. 3 | package resolve 4 | -------------------------------------------------------------------------------- /v2/pkg/resolve/resolve.go: -------------------------------------------------------------------------------- 1 | package resolve 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/rs/xid" 8 | ) 9 | 10 | const ( 11 | maxWildcardChecks = 3 12 | ) 13 | 14 | // ResolutionPool is a pool of resolvers created for resolving subdomains 15 | // for a given host. 16 | type ResolutionPool struct { 17 | *Resolver 18 | Tasks chan HostEntry 19 | Results chan Result 20 | wg *sync.WaitGroup 21 | removeWildcard bool 22 | 23 | wildcardIPs map[string]struct{} 24 | } 25 | 26 | // HostEntry defines a host with the source 27 | type HostEntry struct { 28 | Domain string 29 | Host string 30 | Source string 31 | } 32 | 33 | // Result contains the result for a host resolution 34 | type Result struct { 35 | Type ResultType 36 | Host string 37 | IP string 38 | Error error 39 | Source string 40 | } 41 | 42 | // ResultType is the type of result found 43 | type ResultType int 44 | 45 | // Types of data result can return 46 | const ( 47 | Subdomain ResultType = iota 48 | Error 49 | ) 50 | 51 | // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain 52 | func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { 53 | resolutionPool := &ResolutionPool{ 54 | Resolver: r, 55 | Tasks: make(chan HostEntry), 56 | Results: make(chan Result), 57 | wg: &sync.WaitGroup{}, 58 | removeWildcard: removeWildcard, 59 | wildcardIPs: make(map[string]struct{}), 60 | } 61 | 62 | go func() { 63 | for i := 0; i < workers; i++ { 64 | resolutionPool.wg.Add(1) 65 | go resolutionPool.resolveWorker() 66 | } 67 | resolutionPool.wg.Wait() 68 | close(resolutionPool.Results) 69 | }() 70 | 71 | return resolutionPool 72 | } 73 | 74 | // InitWildcards inits the wildcard ips array 75 | func (r *ResolutionPool) InitWildcards(domain string) error { 76 | for i := 0; i < maxWildcardChecks; i++ { 77 | uid := xid.New().String() 78 | 79 | hosts, _ := r.DNSClient.Lookup(uid + "." + domain) 80 | if len(hosts) == 0 { 81 | return fmt.Errorf("%s is not a wildcard domain", domain) 82 | } 83 | 84 | // Append all wildcard ips found for domains 85 | for _, host := range hosts { 86 | r.wildcardIPs[host] = struct{}{} 87 | } 88 | } 89 | return nil 90 | } 91 | 92 | func (r *ResolutionPool) resolveWorker() { 93 | for task := range r.Tasks { 94 | if !r.removeWildcard { 95 | r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} 96 | continue 97 | } 98 | 99 | hosts, err := r.DNSClient.Lookup(task.Host) 100 | if err != nil { 101 | r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} 102 | continue 103 | } 104 | 105 | if len(hosts) == 0 { 106 | continue 107 | } 108 | 109 | var skip bool 110 | for _, host := range hosts { 111 | // Ignore the host if it exists in wildcard ips map 112 | if _, ok := r.wildcardIPs[host]; ok { 113 | skip = true 114 | break 115 | } 116 | } 117 | 118 | if !skip { 119 | r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} 120 | } 121 | } 122 | r.wg.Done() 123 | } 124 | -------------------------------------------------------------------------------- /v2/pkg/runner/banners.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/projectdiscovery/gologger" 5 | updateutils "github.com/projectdiscovery/utils/update" 6 | ) 7 | 8 | const banner = ` 9 | __ _____ __ 10 | _______ __/ /_ / __(_)___ ____/ /__ _____ 11 | / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ 12 | (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / 13 | /____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ 14 | ` 15 | 16 | // Name 17 | const ToolName = `subfinder` 18 | 19 | // Version is the current version of subfinder 20 | const version = `v2.7.1` 21 | 22 | // showBanner is used to show the banner to the user 23 | func showBanner() { 24 | gologger.Print().Msgf("%s\n", banner) 25 | gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") 26 | } 27 | 28 | // GetUpdateCallback returns a callback function that updates subfinder 29 | func GetUpdateCallback() func() { 30 | return func() { 31 | showBanner() 32 | updateutils.GetUpdateToolCallback("subfinder", version)() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /v2/pkg/runner/config.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "gopkg.in/yaml.v3" 8 | 9 | "github.com/projectdiscovery/gologger" 10 | "github.com/projectdiscovery/subfinder/v2/pkg/passive" 11 | fileutil "github.com/projectdiscovery/utils/file" 12 | ) 13 | 14 | // createProviderConfigYAML marshals the input map to the given location on the disk 15 | func createProviderConfigYAML(configFilePath string) error { 16 | configFile, err := os.Create(configFilePath) 17 | if err != nil { 18 | return err 19 | } 20 | defer configFile.Close() 21 | 22 | sourcesRequiringApiKeysMap := make(map[string][]string) 23 | for _, source := range passive.AllSources { 24 | if source.NeedsKey() { 25 | sourceName := strings.ToLower(source.Name()) 26 | sourcesRequiringApiKeysMap[sourceName] = []string{} 27 | } 28 | } 29 | 30 | return yaml.NewEncoder(configFile).Encode(sourcesRequiringApiKeysMap) 31 | } 32 | 33 | // UnmarshalFrom writes the marshaled yaml config to disk 34 | func UnmarshalFrom(file string) error { 35 | reader, err := fileutil.SubstituteConfigFromEnvVars(file) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | sourceApiKeysMap := map[string][]string{} 41 | err = yaml.NewDecoder(reader).Decode(sourceApiKeysMap) 42 | for _, source := range passive.AllSources { 43 | sourceName := strings.ToLower(source.Name()) 44 | apiKeys := sourceApiKeysMap[sourceName] 45 | if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 { 46 | gologger.Debug().Msgf("API key(s) found for %s.", sourceName) 47 | source.AddApiKeys(apiKeys) 48 | } 49 | } 50 | return err 51 | } 52 | -------------------------------------------------------------------------------- /v2/pkg/runner/doc.go: -------------------------------------------------------------------------------- 1 | // Package runner implements the mechanism to drive the 2 | // subdomain enumeration process 3 | package runner 4 | -------------------------------------------------------------------------------- /v2/pkg/runner/enumerate_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestFilterAndMatchSubdomain(t *testing.T) { 11 | options := &Options{} 12 | options.Domain = []string{"example.com"} 13 | options.Threads = 10 14 | options.Timeout = 10 15 | options.Output = os.Stdout 16 | t.Run("Literal Match", func(t *testing.T) { 17 | options.Match = []string{"req.example.com"} 18 | err := options.validateOptions() 19 | if err != nil { 20 | t.Fatalf("Expected nil got %v while validation\n", err) 21 | } 22 | runner, err := NewRunner(options) 23 | if err != nil { 24 | t.Fatalf("Expected nil got %v while creating runner\n", err) 25 | } 26 | match := runner.filterAndMatchSubdomain("req.example.com") 27 | require.True(t, match, "Expecting a boolean True value ") 28 | }) 29 | t.Run("Multiple Wildcards Match", func(t *testing.T) { 30 | options.Match = []string{"*.ns.*.com"} 31 | err := options.validateOptions() 32 | if err != nil { 33 | t.Fatalf("Expected nil got %v while validation\n", err) 34 | } 35 | runner, err := NewRunner(options) 36 | if err != nil { 37 | t.Fatalf("Expected nil got %v while creating runner\n", err) 38 | } 39 | subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"} 40 | for _, sub := range subdomain { 41 | match := runner.filterAndMatchSubdomain(sub) 42 | require.True(t, match, "Expecting a boolean True value ") 43 | } 44 | }) 45 | t.Run("Sequential Match", func(t *testing.T) { 46 | options.Match = []string{"*.ns.example.com", "*.hackerone.com"} 47 | err := options.validateOptions() 48 | if err != nil { 49 | t.Fatalf("Expected nil got %v while validation\n", err) 50 | } 51 | runner, err := NewRunner(options) 52 | if err != nil { 53 | t.Fatalf("Expected nil got %v while creating runner\n", err) 54 | } 55 | subdomain := []string{"a.ns.example.com", "b.hackerone.com"} 56 | for _, sub := range subdomain { 57 | match := runner.filterAndMatchSubdomain(sub) 58 | require.True(t, match, "Expecting a boolean True value ") 59 | } 60 | }) 61 | t.Run("Literal Filter", func(t *testing.T) { 62 | options.Filter = []string{"req.example.com"} 63 | err := options.validateOptions() 64 | if err != nil { 65 | t.Fatalf("Expected nil got %v while validation\n", err) 66 | } 67 | runner, err := NewRunner(options) 68 | if err != nil { 69 | t.Fatalf("Expected nil got %v while creating runner\n", err) 70 | } 71 | match := runner.filterAndMatchSubdomain("req.example.com") 72 | require.False(t, match, "Expecting a boolean False value ") 73 | }) 74 | t.Run("Multiple Wildcards Filter", func(t *testing.T) { 75 | options.Filter = []string{"*.ns.*.com"} 76 | err := options.validateOptions() 77 | if err != nil { 78 | t.Fatalf("Expected nil got %v while validation\n", err) 79 | } 80 | runner, err := NewRunner(options) 81 | if err != nil { 82 | t.Fatalf("Expected nil got %v while creating runner\n", err) 83 | } 84 | subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"} 85 | for _, sub := range subdomain { 86 | match := runner.filterAndMatchSubdomain(sub) 87 | require.False(t, match, "Expecting a boolean False value ") 88 | } 89 | }) 90 | t.Run("Sequential Filter", func(t *testing.T) { 91 | options.Filter = []string{"*.ns.example.com", "*.hackerone.com"} 92 | err := options.validateOptions() 93 | if err != nil { 94 | t.Fatalf("Expected nil got %v while validation\n", err) 95 | } 96 | runner, err := NewRunner(options) 97 | if err != nil { 98 | t.Fatalf("Expected nil got %v while creating runner\n", err) 99 | } 100 | subdomain := []string{"a.ns.example.com", "b.hackerone.com"} 101 | for _, sub := range subdomain { 102 | match := runner.filterAndMatchSubdomain(sub) 103 | require.False(t, match, "Expecting a boolean False value ") 104 | } 105 | }) 106 | t.Run("Filter and Match", func(t *testing.T) { 107 | options.Filter = []string{"example.com"} 108 | options.Match = []string{"hackerone.com"} 109 | err := options.validateOptions() 110 | if err != nil { 111 | t.Fatalf("Expected nil got %v while validation\n", err) 112 | } 113 | runner, err := NewRunner(options) 114 | if err != nil { 115 | t.Fatalf("Expected nil got %v while creating runner\n", err) 116 | } 117 | subdomain := []string{"example.com", "example.com"} 118 | for _, sub := range subdomain { 119 | match := runner.filterAndMatchSubdomain(sub) 120 | require.False(t, match, "Expecting a boolean False value ") 121 | } 122 | }) 123 | 124 | t.Run("Filter and Match - Same Root Domain", func(t *testing.T) { 125 | options.Filter = []string{"example.com"} 126 | options.Match = []string{"www.example.com"} 127 | err := options.validateOptions() 128 | if err != nil { 129 | t.Fatalf("Expected nil got %v while validation\n", err) 130 | } 131 | runner, err := NewRunner(options) 132 | if err != nil { 133 | t.Fatalf("Expected nil got %v while creating runner\n", err) 134 | } 135 | subdomain := map[string]string{"filter": "example.com", "match": "www.example.com"} 136 | for key, sub := range subdomain { 137 | result := runner.filterAndMatchSubdomain(sub) 138 | if key == "filter" { 139 | require.False(t, result, "Expecting a boolean False value ") 140 | } else { 141 | require.True(t, result, "Expecting a boolean True value ") 142 | } 143 | } 144 | }) 145 | } 146 | -------------------------------------------------------------------------------- /v2/pkg/runner/initialize.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | 7 | "github.com/projectdiscovery/dnsx/libs/dnsx" 8 | "github.com/projectdiscovery/subfinder/v2/pkg/passive" 9 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve" 10 | ) 11 | 12 | // initializePassiveEngine creates the passive engine and loads sources etc 13 | func (r *Runner) initializePassiveEngine() { 14 | r.passiveAgent = passive.New(r.options.Sources, r.options.ExcludeSources, r.options.All, r.options.OnlyRecursive) 15 | } 16 | 17 | // initializeResolver creates the resolver used to resolve the found subdomains 18 | func (r *Runner) initializeResolver() error { 19 | var resolvers []string 20 | 21 | // If the file has been provided, read resolvers from the file 22 | if r.options.ResolverList != "" { 23 | var err error 24 | resolvers, err = loadFromFile(r.options.ResolverList) 25 | if err != nil { 26 | return err 27 | } 28 | } 29 | 30 | if len(r.options.Resolvers) > 0 { 31 | resolvers = append(resolvers, r.options.Resolvers...) 32 | } else { 33 | resolvers = append(resolvers, resolve.DefaultResolvers...) 34 | } 35 | 36 | // Add default 53 UDP port if missing 37 | for i, resolver := range resolvers { 38 | if !strings.Contains(resolver, ":") { 39 | resolvers[i] = net.JoinHostPort(resolver, "53") 40 | } 41 | } 42 | 43 | r.resolverClient = resolve.New() 44 | var err error 45 | r.resolverClient.DNSClient, err = dnsx.New(dnsx.Options{BaseResolvers: resolvers, MaxRetries: 5}) 46 | if err != nil { 47 | return nil 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /v2/pkg/runner/stats.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | "time" 8 | 9 | "github.com/projectdiscovery/gologger" 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | "golang.org/x/exp/maps" 12 | ) 13 | 14 | func printStatistics(stats map[string]subscraping.Statistics) { 15 | 16 | sources := maps.Keys(stats) 17 | sort.Strings(sources) 18 | 19 | var lines []string 20 | var skipped []string 21 | 22 | for _, source := range sources { 23 | sourceStats := stats[source] 24 | if sourceStats.Skipped { 25 | skipped = append(skipped, fmt.Sprintf(" %s", source)) 26 | } else { 27 | lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors)) 28 | } 29 | } 30 | 31 | if len(lines) > 0 { 32 | gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56)) 33 | gologger.Print().Msg(strings.Join(lines, "\n")) 34 | gologger.Print().Msgf("\n") 35 | } 36 | 37 | if len(skipped) > 0 { 38 | gologger.Print().Msgf("\n The following sources were included but skipped...\n\n") 39 | gologger.Print().Msg(strings.Join(skipped, "\n")) 40 | gologger.Print().Msgf("\n\n") 41 | } 42 | } 43 | 44 | func (r *Runner) GetStatistics() map[string]subscraping.Statistics { 45 | return r.passiveAgent.GetStatistics() 46 | } 47 | -------------------------------------------------------------------------------- /v2/pkg/runner/util.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | fileutil "github.com/projectdiscovery/utils/file" 5 | stringsutil "github.com/projectdiscovery/utils/strings" 6 | ) 7 | 8 | func loadFromFile(file string) ([]string, error) { 9 | chanItems, err := fileutil.ReadFile(file) 10 | if err != nil { 11 | return nil, err 12 | } 13 | var items []string 14 | for item := range chanItems { 15 | item = preprocessDomain(item) 16 | if item == "" { 17 | continue 18 | } 19 | items = append(items, item) 20 | } 21 | return items, nil 22 | } 23 | 24 | func preprocessDomain(s string) string { 25 | return stringsutil.NormalizeWithOptions(s, 26 | stringsutil.NormalizeOptions{ 27 | StripComments: true, 28 | TrimCutset: "\n\t\"'` ", 29 | Lowercase: true, 30 | }, 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /v2/pkg/runner/validate.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/projectdiscovery/gologger" 10 | "github.com/projectdiscovery/gologger/formatter" 11 | "github.com/projectdiscovery/gologger/levels" 12 | "github.com/projectdiscovery/subfinder/v2/pkg/passive" 13 | mapsutil "github.com/projectdiscovery/utils/maps" 14 | sliceutil "github.com/projectdiscovery/utils/slice" 15 | ) 16 | 17 | // validateOptions validates the configuration options passed 18 | func (options *Options) validateOptions() error { 19 | // Check if domain, list of domains, or stdin info was provided. 20 | // If none was provided, then return. 21 | if len(options.Domain) == 0 && options.DomainsFile == "" && !options.Stdin { 22 | return errors.New("no input list provided") 23 | } 24 | 25 | // Both verbose and silent flags were used 26 | if options.Verbose && options.Silent { 27 | return errors.New("both verbose and silent mode specified") 28 | } 29 | 30 | // Validate threads and options 31 | if options.Threads == 0 { 32 | return errors.New("threads cannot be zero") 33 | } 34 | if options.Timeout == 0 { 35 | return errors.New("timeout cannot be zero") 36 | } 37 | 38 | // Always remove wildcard with hostip 39 | if options.HostIP && !options.RemoveWildcard { 40 | return errors.New("hostip flag must be used with RemoveWildcard option") 41 | } 42 | 43 | if options.Match != nil { 44 | options.matchRegexes = make([]*regexp.Regexp, len(options.Match)) 45 | var err error 46 | for i, re := range options.Match { 47 | if options.matchRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil { 48 | return errors.New("invalid value for match regex option") 49 | } 50 | } 51 | } 52 | if options.Filter != nil { 53 | options.filterRegexes = make([]*regexp.Regexp, len(options.Filter)) 54 | var err error 55 | for i, re := range options.Filter { 56 | if options.filterRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil { 57 | return errors.New("invalid value for filter regex option") 58 | } 59 | } 60 | } 61 | 62 | sources := mapsutil.GetKeys(passive.NameSourceMap) 63 | for source := range options.RateLimits.AsMap() { 64 | if !sliceutil.Contains(sources, source) { 65 | return fmt.Errorf("invalid source %s specified in -rls flag", source) 66 | } 67 | } 68 | return nil 69 | } 70 | func stripRegexString(val string) string { 71 | val = strings.ReplaceAll(val, ".", "\\.") 72 | val = strings.ReplaceAll(val, "*", ".*") 73 | return fmt.Sprint("^", val, "$") 74 | } 75 | 76 | // ConfigureOutput configures the output on the screen 77 | func (options *Options) ConfigureOutput() { 78 | // If the user desires verbose output, show verbose output 79 | if options.Verbose { 80 | gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) 81 | } 82 | if options.NoColor { 83 | gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) 84 | } 85 | if options.Silent { 86 | gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/agent.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/tls" 7 | "fmt" 8 | "io" 9 | "net" 10 | "net/http" 11 | "net/url" 12 | "time" 13 | 14 | "github.com/corpix/uarand" 15 | "github.com/projectdiscovery/ratelimit" 16 | 17 | "github.com/projectdiscovery/gologger" 18 | ) 19 | 20 | // NewSession creates a new session object for a domain 21 | func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLimiter, timeout int) (*Session, error) { 22 | Transport := &http.Transport{ 23 | MaxIdleConns: 100, 24 | MaxIdleConnsPerHost: 100, 25 | TLSClientConfig: &tls.Config{ 26 | InsecureSkipVerify: true, 27 | }, 28 | Dial: (&net.Dialer{ 29 | Timeout: time.Duration(timeout) * time.Second, 30 | }).Dial, 31 | } 32 | 33 | // Add proxy 34 | if proxy != "" { 35 | proxyURL, _ := url.Parse(proxy) 36 | if proxyURL == nil { 37 | // Log warning but continue anyway 38 | gologger.Warning().Msgf("Invalid proxy provided: %s", proxy) 39 | } else { 40 | Transport.Proxy = http.ProxyURL(proxyURL) 41 | } 42 | } 43 | 44 | client := &http.Client{ 45 | Transport: Transport, 46 | Timeout: time.Duration(timeout) * time.Second, 47 | } 48 | 49 | session := &Session{Client: client} 50 | 51 | // Initiate rate limit instance 52 | session.MultiRateLimiter = multiRateLimiter 53 | 54 | // Create a new extractor object for the current domain 55 | extractor, err := NewSubdomainExtractor(domain) 56 | session.Extractor = extractor 57 | 58 | return session, err 59 | } 60 | 61 | // Get makes a GET request to a URL with extended parameters 62 | func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) { 63 | return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{}) 64 | } 65 | 66 | // SimpleGet makes a simple GET request to a URL 67 | func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) { 68 | return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{}) 69 | } 70 | 71 | // Post makes a POST request to a URL with extended parameters 72 | func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { 73 | return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{}) 74 | } 75 | 76 | // SimplePost makes a simple POST request to a URL 77 | func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) { 78 | return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{}) 79 | } 80 | 81 | // HTTPRequest makes any HTTP request to a URL with extended parameters 82 | func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) { 83 | req, err := http.NewRequestWithContext(ctx, method, requestURL, body) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | req.Header.Set("User-Agent", uarand.GetRandom()) 89 | req.Header.Set("Accept", "*/*") 90 | req.Header.Set("Accept-Language", "en") 91 | req.Header.Set("Connection", "close") 92 | 93 | if basicAuth.Username != "" || basicAuth.Password != "" { 94 | req.SetBasicAuth(basicAuth.Username, basicAuth.Password) 95 | } 96 | 97 | if cookies != "" { 98 | req.Header.Set("Cookie", cookies) 99 | } 100 | 101 | for key, value := range headers { 102 | req.Header.Set(key, value) 103 | } 104 | 105 | sourceName := ctx.Value(CtxSourceArg).(string) 106 | mrlErr := s.MultiRateLimiter.Take(sourceName) 107 | if mrlErr != nil { 108 | return nil, mrlErr 109 | } 110 | 111 | return httpRequestWrapper(s.Client, req) 112 | } 113 | 114 | // DiscardHTTPResponse discards the response content by demand 115 | func (s *Session) DiscardHTTPResponse(response *http.Response) { 116 | if response != nil { 117 | _, err := io.Copy(io.Discard, response.Body) 118 | if err != nil { 119 | gologger.Warning().Msgf("Could not discard response body: %s\n", err) 120 | return 121 | } 122 | response.Body.Close() 123 | } 124 | } 125 | 126 | // Close the session 127 | func (s *Session) Close() { 128 | s.MultiRateLimiter.Stop() 129 | s.Client.CloseIdleConnections() 130 | } 131 | 132 | func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { 133 | response, err := client.Do(request) 134 | if err != nil { 135 | return nil, err 136 | } 137 | 138 | if response.StatusCode != http.StatusOK { 139 | requestURL, _ := url.QueryUnescape(request.URL.String()) 140 | 141 | gologger.Debug().MsgFunc(func() string { 142 | buffer := new(bytes.Buffer) 143 | _, _ = buffer.ReadFrom(response.Body) 144 | return fmt.Sprintf("Response for failed request against %s:\n%s", requestURL, buffer.String()) 145 | }) 146 | return response, fmt.Errorf("unexpected status code %d received from %s", response.StatusCode, requestURL) 147 | } 148 | return response, nil 149 | } 150 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/doc.go: -------------------------------------------------------------------------------- 1 | // Package subscraping contains the logic of scraping agents 2 | package subscraping 3 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/extractor.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | // RegexSubdomainExtractor is a concrete implementation of the SubdomainExtractor interface, using regex for extraction. 9 | type RegexSubdomainExtractor struct { 10 | extractor *regexp.Regexp 11 | } 12 | 13 | // NewSubdomainExtractor creates a new regular expression to extract 14 | // subdomains from text based on the given domain. 15 | func NewSubdomainExtractor(domain string) (*RegexSubdomainExtractor, error) { 16 | extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return &RegexSubdomainExtractor{extractor: extractor}, nil 21 | } 22 | 23 | // Extract implements the SubdomainExtractor interface, using the regex to find subdomains in the given text. 24 | func (re *RegexSubdomainExtractor) Extract(text string) []string { 25 | matches := re.extractor.FindAllString(text, -1) 26 | for i, match := range matches { 27 | matches[i] = strings.ToLower(match) 28 | } 29 | return matches 30 | } 31 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/alienvault/alienvault.go: -------------------------------------------------------------------------------- 1 | // Package alienvault logic 2 | package alienvault 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | type alienvaultResponse struct { 14 | Detail string `json:"detail"` 15 | Error string `json:"error"` 16 | PassiveDNS []struct { 17 | Hostname string `json:"hostname"` 18 | } `json:"passive_dns"` 19 | } 20 | 21 | // Source is the passive scraping agent 22 | type Source struct { 23 | timeTaken time.Duration 24 | results int 25 | errors int 26 | } 27 | 28 | // Run function returns all subdomains found with the service 29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 30 | results := make(chan subscraping.Result) 31 | s.errors = 0 32 | s.results = 0 33 | 34 | go func() { 35 | defer func(startTime time.Time) { 36 | s.timeTaken = time.Since(startTime) 37 | close(results) 38 | }(time.Now()) 39 | 40 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) 41 | if err != nil && resp == nil { 42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 43 | s.errors++ 44 | session.DiscardHTTPResponse(resp) 45 | return 46 | } 47 | 48 | var response alienvaultResponse 49 | // Get the response body and decode 50 | err = json.NewDecoder(resp.Body).Decode(&response) 51 | if err != nil { 52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 53 | s.errors++ 54 | resp.Body.Close() 55 | return 56 | } 57 | resp.Body.Close() 58 | 59 | if response.Error != "" { 60 | results <- subscraping.Result{ 61 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error), 62 | } 63 | return 64 | } 65 | 66 | for _, record := range response.PassiveDNS { 67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} 68 | s.results++ 69 | } 70 | }() 71 | 72 | return results 73 | } 74 | 75 | // Name returns the name of the source 76 | func (s *Source) Name() string { 77 | return "alienvault" 78 | } 79 | 80 | func (s *Source) IsDefault() bool { 81 | return true 82 | } 83 | 84 | func (s *Source) HasRecursiveSupport() bool { 85 | return true 86 | } 87 | 88 | func (s *Source) NeedsKey() bool { 89 | return false 90 | } 91 | 92 | func (s *Source) AddApiKeys(_ []string) { 93 | // no key needed 94 | } 95 | 96 | func (s *Source) Statistics() subscraping.Statistics { 97 | return subscraping.Statistics{ 98 | Errors: s.errors, 99 | Results: s.results, 100 | TimeTaken: s.timeTaken, 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/anubis/anubis.go: -------------------------------------------------------------------------------- 1 | // Package anubis logic 2 | package anubis 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | // Source is the passive scraping agent 16 | type Source struct { 17 | timeTaken time.Duration 18 | errors int 19 | results int 20 | } 21 | 22 | // Run function returns all subdomains found with the service 23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 24 | results := make(chan subscraping.Result) 25 | s.errors = 0 26 | s.results = 0 27 | 28 | go func() { 29 | defer func(startTime time.Time) { 30 | s.timeTaken = time.Since(startTime) 31 | close(results) 32 | }(time.Now()) 33 | 34 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) 35 | if err != nil { 36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 37 | s.errors++ 38 | session.DiscardHTTPResponse(resp) 39 | return 40 | } 41 | 42 | if resp.StatusCode != http.StatusOK { 43 | resp.Body.Close() 44 | return 45 | } 46 | 47 | var subdomains []string 48 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) 49 | if err != nil { 50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 51 | s.errors++ 52 | resp.Body.Close() 53 | return 54 | } 55 | 56 | resp.Body.Close() 57 | 58 | for _, record := range subdomains { 59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} 60 | s.results++ 61 | } 62 | 63 | }() 64 | 65 | return results 66 | } 67 | 68 | // Name returns the name of the source 69 | func (s *Source) Name() string { 70 | return "anubis" 71 | } 72 | 73 | func (s *Source) IsDefault() bool { 74 | return true 75 | } 76 | 77 | func (s *Source) HasRecursiveSupport() bool { 78 | return false 79 | } 80 | 81 | func (s *Source) NeedsKey() bool { 82 | return false 83 | } 84 | 85 | func (s *Source) AddApiKeys(_ []string) { 86 | // no key needed 87 | } 88 | 89 | func (s *Source) Statistics() subscraping.Statistics { 90 | return subscraping.Statistics{ 91 | Errors: s.errors, 92 | Results: s.results, 93 | TimeTaken: s.timeTaken, 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/bevigil/bevigil.go: -------------------------------------------------------------------------------- 1 | // Package bevigil logic 2 | package bevigil 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | type Response struct { 15 | Domain string `json:"domain"` 16 | Subdomains []string `json:"subdomains"` 17 | } 18 | 19 | type Source struct { 20 | apiKeys []string 21 | timeTaken time.Duration 22 | errors int 23 | results int 24 | skipped bool 25 | } 26 | 27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 28 | results := make(chan subscraping.Result) 29 | s.errors = 0 30 | s.results = 0 31 | 32 | go func() { 33 | defer func(startTime time.Time) { 34 | s.timeTaken = time.Since(startTime) 35 | close(results) 36 | }(time.Now()) 37 | 38 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 39 | if randomApiKey == "" { 40 | s.skipped = true 41 | return 42 | } 43 | 44 | getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) 45 | 46 | resp, err := session.Get(ctx, getUrl, "", map[string]string{ 47 | "X-Access-Token": randomApiKey, "User-Agent": "subfinder", 48 | }) 49 | if err != nil { 50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 51 | s.errors++ 52 | session.DiscardHTTPResponse(resp) 53 | return 54 | } 55 | 56 | var subdomains []string 57 | var response Response 58 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | resp.Body.Close() 63 | return 64 | } 65 | 66 | resp.Body.Close() 67 | 68 | if len(response.Subdomains) > 0 { 69 | subdomains = response.Subdomains 70 | } 71 | 72 | for _, subdomain := range subdomains { 73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 74 | s.results++ 75 | } 76 | 77 | }() 78 | return results 79 | } 80 | 81 | func (s *Source) Name() string { 82 | return "bevigil" 83 | } 84 | 85 | func (s *Source) IsDefault() bool { 86 | return true 87 | } 88 | 89 | func (s *Source) HasRecursiveSupport() bool { 90 | return false 91 | } 92 | 93 | func (s *Source) NeedsKey() bool { 94 | return true 95 | } 96 | 97 | func (s *Source) AddApiKeys(keys []string) { 98 | s.apiKeys = keys 99 | } 100 | 101 | func (s *Source) Statistics() subscraping.Statistics { 102 | return subscraping.Statistics{ 103 | Errors: s.errors, 104 | Results: s.results, 105 | TimeTaken: s.timeTaken, 106 | Skipped: s.skipped, 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/binaryedge/binaryedge.go: -------------------------------------------------------------------------------- 1 | // Package binaryedge logic 2 | package binaryedge 3 | 4 | import ( 5 | "context" 6 | "errors" 7 | "fmt" 8 | "math" 9 | "net/url" 10 | "strconv" 11 | "time" 12 | 13 | jsoniter "github.com/json-iterator/go" 14 | 15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 16 | ) 17 | 18 | const ( 19 | v1 = "v1" 20 | v2 = "v2" 21 | baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" 22 | v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" 23 | v1PageSizeParam = "pagesize" 24 | pageParam = "page" 25 | firstPage = 1 26 | maxV1PageSize = 10000 27 | ) 28 | 29 | type subdomainsResponse struct { 30 | Message string `json:"message"` 31 | Title string `json:"title"` 32 | Status interface{} `json:"status"` // string for v1, int for v2 33 | Subdomains []string `json:"events"` 34 | Page int `json:"page"` 35 | PageSize int `json:"pagesize"` 36 | Total int `json:"total"` 37 | } 38 | 39 | // Source is the passive scraping agent 40 | type Source struct { 41 | apiKeys []string 42 | timeTaken time.Duration 43 | errors int 44 | results int 45 | skipped bool 46 | } 47 | 48 | // Run function returns all subdomains found with the service 49 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 50 | results := make(chan subscraping.Result) 51 | s.errors = 0 52 | s.results = 0 53 | 54 | go func() { 55 | defer func(startTime time.Time) { 56 | s.timeTaken = time.Since(startTime) 57 | close(results) 58 | }(time.Now()) 59 | 60 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 61 | if randomApiKey == "" { 62 | s.skipped = true 63 | return 64 | } 65 | 66 | var baseURL string 67 | 68 | authHeader := map[string]string{"X-Key": randomApiKey} 69 | 70 | if isV2(ctx, session, authHeader) { 71 | baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) 72 | } else { 73 | authHeader = map[string]string{"X-Token": randomApiKey} 74 | v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) 75 | if err != nil { 76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 77 | s.errors++ 78 | return 79 | } 80 | baseURL = v1URLWithPageSize.String() 81 | } 82 | 83 | if baseURL == "" { 84 | results <- subscraping.Result{ 85 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL"), 86 | } 87 | s.errors++ 88 | return 89 | } 90 | 91 | s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) 92 | }() 93 | return results 94 | } 95 | 96 | func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { 97 | pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) 98 | if err != nil { 99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 100 | s.errors++ 101 | return 102 | } 103 | 104 | resp, err := session.Get(ctx, pageURL.String(), "", authHeader) 105 | if err != nil { 106 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 107 | s.errors++ 108 | session.DiscardHTTPResponse(resp) 109 | return 110 | } 111 | 112 | var response subdomainsResponse 113 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 114 | if err != nil { 115 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 116 | s.errors++ 117 | resp.Body.Close() 118 | return 119 | } 120 | 121 | // Check error messages 122 | if response.Message != "" && response.Status != nil { 123 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New(response.Message)} 124 | s.errors++ 125 | return 126 | } 127 | 128 | resp.Body.Close() 129 | 130 | for _, subdomain := range response.Subdomains { 131 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 132 | s.results++ 133 | } 134 | 135 | totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) 136 | nextPage := response.Page + 1 137 | if nextPage <= totalPages { 138 | s.enumerate(ctx, session, baseURL, nextPage, authHeader, results) 139 | } 140 | } 141 | 142 | // Name returns the name of the source 143 | func (s *Source) Name() string { 144 | return "binaryedge" 145 | } 146 | 147 | func (s *Source) IsDefault() bool { 148 | return false 149 | } 150 | 151 | func (s *Source) HasRecursiveSupport() bool { 152 | return true 153 | } 154 | 155 | func (s *Source) NeedsKey() bool { 156 | return true 157 | } 158 | 159 | func (s *Source) AddApiKeys(keys []string) { 160 | s.apiKeys = keys 161 | } 162 | 163 | func (s *Source) Statistics() subscraping.Statistics { 164 | return subscraping.Statistics{ 165 | Errors: s.errors, 166 | Results: s.results, 167 | TimeTaken: s.timeTaken, 168 | Skipped: s.skipped, 169 | } 170 | } 171 | 172 | func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { 173 | resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) 174 | if err != nil { 175 | session.DiscardHTTPResponse(resp) 176 | return false 177 | } 178 | 179 | resp.Body.Close() 180 | 181 | return true 182 | } 183 | 184 | func addURLParam(targetURL, name, value string) (*url.URL, error) { 185 | u, err := url.Parse(targetURL) 186 | if err != nil { 187 | return u, err 188 | } 189 | q, _ := url.ParseQuery(u.RawQuery) 190 | q.Add(name, value) 191 | u.RawQuery = q.Encode() 192 | 193 | return u, nil 194 | } 195 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/bufferover/bufferover.go: -------------------------------------------------------------------------------- 1 | // Package bufferover is a bufferover Scraping Engine in Golang 2 | package bufferover 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | type response struct { 16 | Meta struct { 17 | Errors []string `json:"Errors"` 18 | } `json:"Meta"` 19 | FDNSA []string `json:"FDNS_A"` 20 | RDNS []string `json:"RDNS"` 21 | Results []string `json:"Results"` 22 | } 23 | 24 | // Source is the passive scraping agent 25 | type Source struct { 26 | apiKeys []string 27 | timeTaken time.Duration 28 | errors int 29 | results int 30 | skipped bool 31 | } 32 | 33 | // Run function returns all subdomains found with the service 34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 35 | results := make(chan subscraping.Result) 36 | s.errors = 0 37 | s.results = 0 38 | 39 | go func() { 40 | defer func(startTime time.Time) { 41 | s.timeTaken = time.Since(startTime) 42 | close(results) 43 | }(time.Now()) 44 | 45 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 46 | if randomApiKey == "" { 47 | s.skipped = true 48 | return 49 | } 50 | 51 | s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), randomApiKey, session, results) 52 | }() 53 | 54 | return results 55 | } 56 | 57 | func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { 58 | resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) 59 | 60 | if err != nil && resp == nil { 61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 62 | s.errors++ 63 | session.DiscardHTTPResponse(resp) 64 | return 65 | } 66 | 67 | var bufforesponse response 68 | err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse) 69 | if err != nil { 70 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 71 | s.errors++ 72 | resp.Body.Close() 73 | return 74 | } 75 | 76 | resp.Body.Close() 77 | 78 | metaErrors := bufforesponse.Meta.Errors 79 | 80 | if len(metaErrors) > 0 { 81 | results <- subscraping.Result{ 82 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", ")), 83 | } 84 | s.errors++ 85 | return 86 | } 87 | 88 | var subdomains []string 89 | 90 | if len(bufforesponse.FDNSA) > 0 { 91 | subdomains = bufforesponse.FDNSA 92 | subdomains = append(subdomains, bufforesponse.RDNS...) 93 | } else if len(bufforesponse.Results) > 0 { 94 | subdomains = bufforesponse.Results 95 | } 96 | 97 | for _, subdomain := range subdomains { 98 | for _, value := range session.Extractor.Extract(subdomain) { 99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} 100 | s.results++ 101 | } 102 | } 103 | } 104 | 105 | // Name returns the name of the source 106 | func (s *Source) Name() string { 107 | return "bufferover" 108 | } 109 | 110 | func (s *Source) IsDefault() bool { 111 | return true 112 | } 113 | 114 | func (s *Source) HasRecursiveSupport() bool { 115 | return true 116 | } 117 | 118 | func (s *Source) NeedsKey() bool { 119 | return true 120 | } 121 | 122 | func (s *Source) AddApiKeys(keys []string) { 123 | s.apiKeys = keys 124 | } 125 | 126 | func (s *Source) Statistics() subscraping.Statistics { 127 | return subscraping.Statistics{ 128 | Errors: s.errors, 129 | Results: s.results, 130 | TimeTaken: s.timeTaken, 131 | Skipped: s.skipped, 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/builtwith/builtwith.go: -------------------------------------------------------------------------------- 1 | // Package builtwith logic 2 | package builtwith 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | type response struct { 15 | Results []resultItem `json:"Results"` 16 | } 17 | 18 | type resultItem struct { 19 | Result result `json:"Result"` 20 | } 21 | 22 | type result struct { 23 | Paths []path `json:"Paths"` 24 | } 25 | 26 | type path struct { 27 | Domain string `json:"Domain"` 28 | Url string `json:"Url"` 29 | SubDomain string `json:"SubDomain"` 30 | } 31 | 32 | // Source is the passive scraping agent 33 | type Source struct { 34 | apiKeys []string 35 | timeTaken time.Duration 36 | errors int 37 | results int 38 | skipped bool 39 | } 40 | 41 | // Run function returns all subdomains found with the service 42 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 43 | results := make(chan subscraping.Result) 44 | s.errors = 0 45 | s.results = 0 46 | 47 | go func() { 48 | defer func(startTime time.Time) { 49 | s.timeTaken = time.Since(startTime) 50 | close(results) 51 | }(time.Now()) 52 | 53 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 54 | if randomApiKey == "" { 55 | return 56 | } 57 | 58 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain)) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | session.DiscardHTTPResponse(resp) 63 | return 64 | } 65 | 66 | var data response 67 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 68 | if err != nil { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 70 | s.errors++ 71 | resp.Body.Close() 72 | return 73 | } 74 | resp.Body.Close() 75 | for _, result := range data.Results { 76 | for _, path := range result.Result.Paths { 77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)} 78 | s.results++ 79 | } 80 | } 81 | }() 82 | 83 | return results 84 | } 85 | 86 | // Name returns the name of the source 87 | func (s *Source) Name() string { 88 | return "builtwith" 89 | } 90 | 91 | func (s *Source) IsDefault() bool { 92 | return true 93 | } 94 | 95 | func (s *Source) HasRecursiveSupport() bool { 96 | return false 97 | } 98 | 99 | func (s *Source) NeedsKey() bool { 100 | return true 101 | } 102 | 103 | func (s *Source) AddApiKeys(keys []string) { 104 | s.apiKeys = keys 105 | } 106 | 107 | func (s *Source) Statistics() subscraping.Statistics { 108 | return subscraping.Statistics{ 109 | Errors: s.errors, 110 | Results: s.results, 111 | TimeTaken: s.timeTaken, 112 | Skipped: s.skipped, 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/c99/c99.go: -------------------------------------------------------------------------------- 1 | // Package c99 logic 2 | package c99 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | // Source is the passive scraping agent 16 | type Source struct { 17 | apiKeys []string 18 | timeTaken time.Duration 19 | errors int 20 | results int 21 | skipped bool 22 | } 23 | 24 | type dnsdbLookupResponse struct { 25 | Success bool `json:"success"` 26 | Subdomains []struct { 27 | Subdomain string `json:"subdomain"` 28 | IP string `json:"ip"` 29 | Cloudflare bool `json:"cloudflare"` 30 | } `json:"subdomains"` 31 | Error string `json:"error"` 32 | } 33 | 34 | // Run function returns all subdomains found with the service 35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 36 | results := make(chan subscraping.Result) 37 | s.errors = 0 38 | s.results = 0 39 | 40 | go func() { 41 | defer func(startTime time.Time) { 42 | s.timeTaken = time.Since(startTime) 43 | close(results) 44 | }(time.Now()) 45 | 46 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 47 | if randomApiKey == "" { 48 | s.skipped = true 49 | return 50 | } 51 | 52 | searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) 53 | resp, err := session.SimpleGet(ctx, searchURL) 54 | if err != nil { 55 | session.DiscardHTTPResponse(resp) 56 | return 57 | } 58 | 59 | defer resp.Body.Close() 60 | 61 | var response dnsdbLookupResponse 62 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 63 | if err != nil { 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 65 | s.errors++ 66 | return 67 | } 68 | 69 | if response.Error != "" { 70 | results <- subscraping.Result{ 71 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), 72 | } 73 | s.errors++ 74 | return 75 | } 76 | 77 | for _, data := range response.Subdomains { 78 | if !strings.HasPrefix(data.Subdomain, ".") { 79 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} 80 | s.results++ 81 | } 82 | } 83 | }() 84 | 85 | return results 86 | } 87 | 88 | // Name returns the name of the source 89 | func (s *Source) Name() string { 90 | return "c99" 91 | } 92 | 93 | func (s *Source) IsDefault() bool { 94 | return true 95 | } 96 | 97 | func (s *Source) HasRecursiveSupport() bool { 98 | return false 99 | } 100 | 101 | func (s *Source) NeedsKey() bool { 102 | return true 103 | } 104 | 105 | func (s *Source) AddApiKeys(keys []string) { 106 | s.apiKeys = keys 107 | } 108 | 109 | func (s *Source) Statistics() subscraping.Statistics { 110 | return subscraping.Statistics{ 111 | Errors: s.errors, 112 | Results: s.results, 113 | TimeTaken: s.timeTaken, 114 | Skipped: s.skipped, 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/censys/censys.go: -------------------------------------------------------------------------------- 1 | // Package censys logic 2 | package censys 3 | 4 | import ( 5 | "context" 6 | "strconv" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | urlutil "github.com/projectdiscovery/utils/url" 13 | ) 14 | 15 | const ( 16 | maxCensysPages = 10 17 | maxPerPage = 100 18 | ) 19 | 20 | type response struct { 21 | Code int `json:"code"` 22 | Status string `json:"status"` 23 | Result result `json:"result"` 24 | } 25 | 26 | type result struct { 27 | Query string `json:"query"` 28 | Total float64 `json:"total"` 29 | DurationMS int `json:"duration_ms"` 30 | Hits []hit `json:"hits"` 31 | Links links `json:"links"` 32 | } 33 | 34 | type hit struct { 35 | Parsed parsed `json:"parsed"` 36 | Names []string `json:"names"` 37 | FingerprintSha256 string `json:"fingerprint_sha256"` 38 | } 39 | 40 | type parsed struct { 41 | ValidityPeriod validityPeriod `json:"validity_period"` 42 | SubjectDN string `json:"subject_dn"` 43 | IssuerDN string `json:"issuer_dn"` 44 | } 45 | 46 | type validityPeriod struct { 47 | NotAfter string `json:"not_after"` 48 | NotBefore string `json:"not_before"` 49 | } 50 | 51 | type links struct { 52 | Next string `json:"next"` 53 | Prev string `json:"prev"` 54 | } 55 | 56 | // Source is the passive scraping agent 57 | type Source struct { 58 | apiKeys []apiKey 59 | timeTaken time.Duration 60 | errors int 61 | results int 62 | skipped bool 63 | } 64 | 65 | type apiKey struct { 66 | token string 67 | secret string 68 | } 69 | 70 | // Run function returns all subdomains found with the service 71 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 72 | results := make(chan subscraping.Result) 73 | s.errors = 0 74 | s.results = 0 75 | 76 | go func() { 77 | defer func(startTime time.Time) { 78 | s.timeTaken = time.Since(startTime) 79 | close(results) 80 | }(time.Now()) 81 | 82 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 83 | if randomApiKey.token == "" || randomApiKey.secret == "" { 84 | s.skipped = true 85 | return 86 | } 87 | 88 | certSearchEndpoint := "https://search.censys.io/api/v2/certificates/search" 89 | cursor := "" 90 | currentPage := 1 91 | for { 92 | certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint) 93 | if err != nil { 94 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 95 | s.errors++ 96 | return 97 | } 98 | 99 | certSearchEndpointUrl.Params.Add("q", domain) 100 | certSearchEndpointUrl.Params.Add("per_page", strconv.Itoa(maxPerPage)) 101 | if cursor != "" { 102 | certSearchEndpointUrl.Params.Add("cursor", cursor) 103 | } 104 | 105 | resp, err := session.HTTPRequest( 106 | ctx, 107 | "GET", 108 | certSearchEndpointUrl.String(), 109 | "", 110 | nil, 111 | nil, 112 | subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret}, 113 | ) 114 | 115 | if err != nil { 116 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 117 | s.errors++ 118 | session.DiscardHTTPResponse(resp) 119 | return 120 | } 121 | 122 | var censysResponse response 123 | err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) 124 | if err != nil { 125 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 126 | s.errors++ 127 | resp.Body.Close() 128 | return 129 | } 130 | 131 | resp.Body.Close() 132 | 133 | for _, hit := range censysResponse.Result.Hits { 134 | for _, name := range hit.Names { 135 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name} 136 | s.results++ 137 | } 138 | } 139 | 140 | // Exit the censys enumeration if last page is reached 141 | cursor = censysResponse.Result.Links.Next 142 | if cursor == "" || currentPage >= maxCensysPages { 143 | break 144 | } 145 | currentPage++ 146 | } 147 | }() 148 | 149 | return results 150 | } 151 | 152 | // Name returns the name of the source 153 | func (s *Source) Name() string { 154 | return "censys" 155 | } 156 | 157 | func (s *Source) IsDefault() bool { 158 | return true 159 | } 160 | 161 | func (s *Source) HasRecursiveSupport() bool { 162 | return false 163 | } 164 | 165 | func (s *Source) NeedsKey() bool { 166 | return true 167 | } 168 | 169 | func (s *Source) AddApiKeys(keys []string) { 170 | s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { 171 | return apiKey{k, v} 172 | }) 173 | } 174 | 175 | func (s *Source) Statistics() subscraping.Statistics { 176 | return subscraping.Statistics{ 177 | Errors: s.errors, 178 | Results: s.results, 179 | TimeTaken: s.timeTaken, 180 | Skipped: s.skipped, 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/certspotter/certspotter.go: -------------------------------------------------------------------------------- 1 | // Package certspotter logic 2 | package certspotter 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | type certspotterObject struct { 15 | ID string `json:"id"` 16 | DNSNames []string `json:"dns_names"` 17 | } 18 | 19 | // Source is the passive scraping agent 20 | type Source struct { 21 | apiKeys []string 22 | timeTaken time.Duration 23 | errors int 24 | results int 25 | skipped bool 26 | } 27 | 28 | // Run function returns all subdomains found with the service 29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 30 | results := make(chan subscraping.Result) 31 | s.errors = 0 32 | s.results = 0 33 | 34 | go func() { 35 | defer func(startTime time.Time) { 36 | s.timeTaken = time.Since(startTime) 37 | close(results) 38 | }(time.Now()) 39 | 40 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 41 | if randomApiKey == "" { 42 | s.skipped = true 43 | return 44 | } 45 | 46 | headers := map[string]string{"Authorization": "Bearer " + randomApiKey} 47 | cookies := "" 48 | 49 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) 50 | if err != nil { 51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 52 | s.errors++ 53 | session.DiscardHTTPResponse(resp) 54 | return 55 | } 56 | 57 | var response []certspotterObject 58 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | resp.Body.Close() 63 | return 64 | } 65 | resp.Body.Close() 66 | 67 | for _, cert := range response { 68 | for _, subdomain := range cert.DNSNames { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 70 | s.results++ 71 | } 72 | } 73 | 74 | // if the number of responses is zero, close the channel and return. 75 | if len(response) == 0 { 76 | return 77 | } 78 | 79 | id := response[len(response)-1].ID 80 | for { 81 | reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) 82 | 83 | resp, err := session.Get(ctx, reqURL, cookies, headers) 84 | if err != nil { 85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 86 | s.errors++ 87 | return 88 | } 89 | 90 | var response []certspotterObject 91 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 92 | if err != nil { 93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 94 | s.errors++ 95 | resp.Body.Close() 96 | return 97 | } 98 | resp.Body.Close() 99 | 100 | if len(response) == 0 { 101 | break 102 | } 103 | 104 | for _, cert := range response { 105 | for _, subdomain := range cert.DNSNames { 106 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 107 | s.results++ 108 | } 109 | } 110 | 111 | id = response[len(response)-1].ID 112 | } 113 | }() 114 | 115 | return results 116 | } 117 | 118 | // Name returns the name of the source 119 | func (s *Source) Name() string { 120 | return "certspotter" 121 | } 122 | 123 | func (s *Source) IsDefault() bool { 124 | return true 125 | } 126 | 127 | func (s *Source) HasRecursiveSupport() bool { 128 | return true 129 | } 130 | 131 | func (s *Source) NeedsKey() bool { 132 | return true 133 | } 134 | 135 | func (s *Source) AddApiKeys(keys []string) { 136 | s.apiKeys = keys 137 | } 138 | 139 | func (s *Source) Statistics() subscraping.Statistics { 140 | return subscraping.Statistics{ 141 | Errors: s.errors, 142 | Results: s.results, 143 | TimeTaken: s.timeTaken, 144 | Skipped: s.skipped, 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/chaos/chaos.go: -------------------------------------------------------------------------------- 1 | // Package chaos logic 2 | package chaos 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/projectdiscovery/chaos-client/pkg/chaos" 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct { 15 | apiKeys []string 16 | timeTaken time.Duration 17 | errors int 18 | results int 19 | skipped bool 20 | } 21 | 22 | // Run function returns all subdomains found with the service 23 | func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { 24 | results := make(chan subscraping.Result) 25 | s.errors = 0 26 | s.results = 0 27 | 28 | go func() { 29 | defer func(startTime time.Time) { 30 | s.timeTaken = time.Since(startTime) 31 | close(results) 32 | }(time.Now()) 33 | 34 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 35 | if randomApiKey == "" { 36 | s.skipped = true 37 | return 38 | } 39 | 40 | chaosClient := chaos.New(randomApiKey) 41 | for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ 42 | Domain: domain, 43 | }) { 44 | if result.Error != nil { 45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} 46 | s.errors++ 47 | break 48 | } 49 | results <- subscraping.Result{ 50 | Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain), 51 | } 52 | s.results++ 53 | } 54 | }() 55 | 56 | return results 57 | } 58 | 59 | // Name returns the name of the source 60 | func (s *Source) Name() string { 61 | return "chaos" 62 | } 63 | 64 | func (s *Source) IsDefault() bool { 65 | return true 66 | } 67 | 68 | func (s *Source) HasRecursiveSupport() bool { 69 | return false 70 | } 71 | 72 | func (s *Source) NeedsKey() bool { 73 | return true 74 | } 75 | 76 | func (s *Source) AddApiKeys(keys []string) { 77 | s.apiKeys = keys 78 | } 79 | 80 | func (s *Source) Statistics() subscraping.Statistics { 81 | return subscraping.Statistics{ 82 | Errors: s.errors, 83 | Results: s.results, 84 | TimeTaken: s.timeTaken, 85 | Skipped: s.skipped, 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/chinaz/chinaz.go: -------------------------------------------------------------------------------- 1 | package chinaz 2 | 3 | // chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | // Source is the passive scraping agent 15 | type Source struct { 16 | apiKeys []string 17 | timeTaken time.Duration 18 | errors int 19 | results int 20 | skipped bool 21 | } 22 | 23 | // Run function returns all subdomains found with the service 24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 25 | results := make(chan subscraping.Result) 26 | s.errors = 0 27 | s.results = 0 28 | 29 | go func() { 30 | defer func(startTime time.Time) { 31 | s.timeTaken = time.Since(startTime) 32 | close(results) 33 | }(time.Now()) 34 | 35 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 36 | if randomApiKey == "" { 37 | s.skipped = true 38 | return 39 | } 40 | 41 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) 42 | if err != nil { 43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 44 | s.errors++ 45 | session.DiscardHTTPResponse(resp) 46 | return 47 | } 48 | 49 | body, err := io.ReadAll(resp.Body) 50 | 51 | resp.Body.Close() 52 | 53 | SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") 54 | 55 | if SubdomainList.ToBool() { 56 | _data := []byte(SubdomainList.ToString()) 57 | for i := 0; i < SubdomainList.Size(); i++ { 58 | subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() 59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 60 | s.results++ 61 | } 62 | } else { 63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 64 | s.errors++ 65 | return 66 | } 67 | }() 68 | 69 | return results 70 | } 71 | 72 | // Name returns the name of the source 73 | func (s *Source) Name() string { 74 | return "chinaz" 75 | } 76 | 77 | func (s *Source) IsDefault() bool { 78 | return true 79 | } 80 | 81 | func (s *Source) HasRecursiveSupport() bool { 82 | return false 83 | } 84 | 85 | func (s *Source) NeedsKey() bool { 86 | return true 87 | } 88 | 89 | func (s *Source) AddApiKeys(keys []string) { 90 | s.apiKeys = keys 91 | } 92 | 93 | func (s *Source) Statistics() subscraping.Statistics { 94 | return subscraping.Statistics{ 95 | Errors: s.errors, 96 | Results: s.results, 97 | TimeTaken: s.timeTaken, 98 | Skipped: s.skipped, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go: -------------------------------------------------------------------------------- 1 | // Package commoncrawl logic 2 | package commoncrawl 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "net/url" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | jsoniter "github.com/json-iterator/go" 14 | 15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 16 | ) 17 | 18 | const ( 19 | indexURL = "https://index.commoncrawl.org/collinfo.json" 20 | maxYearsBack = 5 21 | ) 22 | 23 | var year = time.Now().Year() 24 | 25 | type indexResponse struct { 26 | ID string `json:"id"` 27 | APIURL string `json:"cdx-api"` 28 | } 29 | 30 | // Source is the passive scraping agent 31 | type Source struct { 32 | timeTaken time.Duration 33 | errors int 34 | results int 35 | } 36 | 37 | // Run function returns all subdomains found with the service 38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 39 | results := make(chan subscraping.Result) 40 | s.errors = 0 41 | s.results = 0 42 | 43 | go func() { 44 | defer func(startTime time.Time) { 45 | s.timeTaken = time.Since(startTime) 46 | close(results) 47 | }(time.Now()) 48 | 49 | resp, err := session.SimpleGet(ctx, indexURL) 50 | if err != nil { 51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 52 | s.errors++ 53 | session.DiscardHTTPResponse(resp) 54 | return 55 | } 56 | 57 | var indexes []indexResponse 58 | err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | resp.Body.Close() 63 | return 64 | } 65 | resp.Body.Close() 66 | 67 | years := make([]string, 0) 68 | for i := 0; i < maxYearsBack; i++ { 69 | years = append(years, strconv.Itoa(year-i)) 70 | } 71 | 72 | searchIndexes := make(map[string]string) 73 | for _, year := range years { 74 | for _, index := range indexes { 75 | if strings.Contains(index.ID, year) { 76 | if _, ok := searchIndexes[year]; !ok { 77 | searchIndexes[year] = index.APIURL 78 | break 79 | } 80 | } 81 | } 82 | } 83 | 84 | for _, apiURL := range searchIndexes { 85 | further := s.getSubdomains(ctx, apiURL, domain, session, results) 86 | if !further { 87 | break 88 | } 89 | } 90 | }() 91 | 92 | return results 93 | } 94 | 95 | // Name returns the name of the source 96 | func (s *Source) Name() string { 97 | return "commoncrawl" 98 | } 99 | 100 | func (s *Source) IsDefault() bool { 101 | return false 102 | } 103 | 104 | func (s *Source) HasRecursiveSupport() bool { 105 | return false 106 | } 107 | 108 | func (s *Source) NeedsKey() bool { 109 | return false 110 | } 111 | 112 | func (s *Source) AddApiKeys(_ []string) { 113 | // no key needed 114 | } 115 | 116 | func (s *Source) Statistics() subscraping.Statistics { 117 | return subscraping.Statistics{ 118 | Errors: s.errors, 119 | Results: s.results, 120 | TimeTaken: s.timeTaken, 121 | } 122 | } 123 | 124 | func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool { 125 | for { 126 | select { 127 | case <-ctx.Done(): 128 | return false 129 | default: 130 | var headers = map[string]string{"Host": "index.commoncrawl.org"} 131 | resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) 132 | if err != nil { 133 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 134 | s.errors++ 135 | session.DiscardHTTPResponse(resp) 136 | return false 137 | } 138 | 139 | scanner := bufio.NewScanner(resp.Body) 140 | for scanner.Scan() { 141 | line := scanner.Text() 142 | if line == "" { 143 | continue 144 | } 145 | line, _ = url.QueryUnescape(line) 146 | for _, subdomain := range session.Extractor.Extract(line) { 147 | if subdomain != "" { 148 | // fix for triple encoded URL 149 | subdomain = strings.ToLower(subdomain) 150 | subdomain = strings.TrimPrefix(subdomain, "25") 151 | subdomain = strings.TrimPrefix(subdomain, "2f") 152 | 153 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 154 | s.results++ 155 | } 156 | } 157 | } 158 | resp.Body.Close() 159 | return true 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/crtsh/crtsh.go: -------------------------------------------------------------------------------- 1 | // Package crtsh logic 2 | package crtsh 3 | 4 | import ( 5 | "context" 6 | "database/sql" 7 | "fmt" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | jsoniter "github.com/json-iterator/go" 13 | 14 | // postgres driver 15 | _ "github.com/lib/pq" 16 | 17 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 18 | contextutil "github.com/projectdiscovery/utils/context" 19 | ) 20 | 21 | type subdomain struct { 22 | ID int `json:"id"` 23 | NameValue string `json:"name_value"` 24 | } 25 | 26 | // Source is the passive scraping agent 27 | type Source struct { 28 | timeTaken time.Duration 29 | errors int 30 | results int 31 | } 32 | 33 | // Run function returns all subdomains found with the service 34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 35 | results := make(chan subscraping.Result) 36 | s.errors = 0 37 | s.results = 0 38 | 39 | go func() { 40 | defer func(startTime time.Time) { 41 | s.timeTaken = time.Since(startTime) 42 | close(results) 43 | }(time.Now()) 44 | 45 | count := s.getSubdomainsFromSQL(ctx, domain, session, results) 46 | if count > 0 { 47 | return 48 | } 49 | _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) 50 | }() 51 | 52 | return results 53 | } 54 | 55 | func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int { 56 | db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") 57 | if err != nil { 58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 59 | s.errors++ 60 | return 0 61 | } 62 | 63 | defer db.Close() 64 | 65 | limitClause := "" 66 | if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok { 67 | if allBool, err := strconv.ParseBool(string(all)); err == nil && !allBool { 68 | limitClause = "LIMIT 10000" 69 | } 70 | } 71 | 72 | query := fmt.Sprintf(`WITH ci AS ( 73 | SELECT min(sub.CERTIFICATE_ID) ID, 74 | min(sub.ISSUER_CA_ID) ISSUER_CA_ID, 75 | array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES, 76 | x509_commonName(sub.CERTIFICATE) COMMON_NAME, 77 | x509_notBefore(sub.CERTIFICATE) NOT_BEFORE, 78 | x509_notAfter(sub.CERTIFICATE) NOT_AFTER, 79 | encode(x509_serialNumber(sub.CERTIFICATE), 'hex') SERIAL_NUMBER 80 | FROM (SELECT * 81 | FROM certificate_and_identities cai 82 | WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE) 83 | AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%') 84 | %s 85 | ) sub 86 | GROUP BY sub.CERTIFICATE 87 | ) 88 | SELECT array_to_string(ci.NAME_VALUES, chr(10)) NAME_VALUE 89 | FROM ci 90 | LEFT JOIN LATERAL ( 91 | SELECT min(ctle.ENTRY_TIMESTAMP) ENTRY_TIMESTAMP 92 | FROM ct_log_entry ctle 93 | WHERE ctle.CERTIFICATE_ID = ci.ID 94 | ) le ON TRUE, 95 | ca 96 | WHERE ci.ISSUER_CA_ID = ca.ID 97 | ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause) 98 | rows, err := db.QueryContext(ctx, query, domain) 99 | if err != nil { 100 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 101 | s.errors++ 102 | return 0 103 | } 104 | if err := rows.Err(); err != nil { 105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 106 | s.errors++ 107 | return 0 108 | } 109 | 110 | var count int 111 | var data string 112 | // Parse all the rows getting subdomains 113 | for rows.Next() { 114 | err := rows.Scan(&data) 115 | if err != nil { 116 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 117 | s.errors++ 118 | return count 119 | } 120 | 121 | count++ 122 | for _, subdomain := range strings.Split(data, "\n") { 123 | for _, value := range session.Extractor.Extract(subdomain) { 124 | if value != "" { 125 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} 126 | s.results++ 127 | } 128 | } 129 | } 130 | } 131 | return count 132 | } 133 | 134 | func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { 135 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) 136 | if err != nil { 137 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 138 | s.errors++ 139 | session.DiscardHTTPResponse(resp) 140 | return false 141 | } 142 | 143 | var subdomains []subdomain 144 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) 145 | if err != nil { 146 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 147 | s.errors++ 148 | resp.Body.Close() 149 | return false 150 | } 151 | 152 | resp.Body.Close() 153 | 154 | for _, subdomain := range subdomains { 155 | for _, sub := range strings.Split(subdomain.NameValue, "\n") { 156 | for _, value := range session.Extractor.Extract(sub) { 157 | if value != "" { 158 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} 159 | s.results++ 160 | } 161 | } 162 | } 163 | } 164 | 165 | return true 166 | } 167 | 168 | // Name returns the name of the source 169 | func (s *Source) Name() string { 170 | return "crtsh" 171 | } 172 | 173 | func (s *Source) IsDefault() bool { 174 | return true 175 | } 176 | 177 | func (s *Source) HasRecursiveSupport() bool { 178 | return true 179 | } 180 | 181 | func (s *Source) NeedsKey() bool { 182 | return false 183 | } 184 | 185 | func (s *Source) AddApiKeys(_ []string) { 186 | // no key needed 187 | } 188 | 189 | func (s *Source) Statistics() subscraping.Statistics { 190 | return subscraping.Statistics{ 191 | Errors: s.errors, 192 | Results: s.results, 193 | TimeTaken: s.timeTaken, 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/digitalyama/digitalyama.go: -------------------------------------------------------------------------------- 1 | package digitalyama 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct { 15 | apiKeys []string 16 | timeTaken time.Duration 17 | errors int 18 | results int 19 | skipped bool 20 | } 21 | 22 | type digitalYamaResponse struct { 23 | Query string `json:"query"` 24 | Count int `json:"count"` 25 | Subdomains []string `json:"subdomains"` 26 | UsageSummary struct { 27 | QueryCost float64 `json:"query_cost"` 28 | CreditsRemaining float64 `json:"credits_remaining"` 29 | } `json:"usage_summary"` 30 | } 31 | 32 | // Run function returns all subdomains found with the service 33 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 34 | results := make(chan subscraping.Result) 35 | s.errors = 0 36 | s.results = 0 37 | 38 | go func() { 39 | defer func(startTime time.Time) { 40 | s.timeTaken = time.Since(startTime) 41 | close(results) 42 | }(time.Now()) 43 | 44 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 45 | if randomApiKey == "" { 46 | s.skipped = true 47 | return 48 | } 49 | 50 | searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) 51 | resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) 52 | if err != nil { 53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 54 | s.errors++ 55 | return 56 | } 57 | defer resp.Body.Close() 58 | 59 | if resp.StatusCode != 200 { 60 | var errResponse struct { 61 | Detail []struct { 62 | Loc []string `json:"loc"` 63 | Msg string `json:"msg"` 64 | Type string `json:"type"` 65 | } `json:"detail"` 66 | } 67 | err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) 68 | if err != nil { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} 70 | s.errors++ 71 | return 72 | } 73 | if len(errResponse.Detail) > 0 { 74 | errMsg := errResponse.Detail[0].Msg 75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)} 76 | } else { 77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} 78 | } 79 | s.errors++ 80 | return 81 | } 82 | 83 | var response digitalYamaResponse 84 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 85 | if err != nil { 86 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 87 | s.errors++ 88 | return 89 | } 90 | 91 | for _, subdomain := range response.Subdomains { 92 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 93 | s.results++ 94 | } 95 | }() 96 | 97 | return results 98 | } 99 | 100 | // Name returns the name of the source 101 | func (s *Source) Name() string { 102 | return "digitalyama" 103 | } 104 | 105 | func (s *Source) IsDefault() bool { 106 | return true 107 | } 108 | 109 | func (s *Source) HasRecursiveSupport() bool { 110 | return false 111 | } 112 | 113 | func (s *Source) NeedsKey() bool { 114 | return true 115 | } 116 | 117 | func (s *Source) AddApiKeys(keys []string) { 118 | s.apiKeys = keys 119 | } 120 | 121 | func (s *Source) Statistics() subscraping.Statistics { 122 | return subscraping.Statistics{ 123 | Errors: s.errors, 124 | Results: s.results, 125 | TimeTaken: s.timeTaken, 126 | Skipped: s.skipped, 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/digitorus/digitorus.go: -------------------------------------------------------------------------------- 1 | // Package waybackarchive logic 2 | package digitorus 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | "github.com/projectdiscovery/utils/ptr" 14 | ) 15 | 16 | // Source is the passive scraping agent 17 | type Source struct { 18 | timeTaken time.Duration 19 | errors int 20 | results int 21 | } 22 | 23 | // Run function returns all subdomains found with the service 24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 25 | results := make(chan subscraping.Result) 26 | s.errors = 0 27 | s.results = 0 28 | 29 | go func() { 30 | defer func(startTime time.Time) { 31 | s.timeTaken = time.Since(startTime) 32 | close(results) 33 | }(time.Now()) 34 | 35 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) 36 | // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 37 | if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound { 38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 39 | s.errors++ 40 | session.DiscardHTTPResponse(resp) 41 | return 42 | } 43 | 44 | defer resp.Body.Close() 45 | 46 | scanner := bufio.NewScanner(resp.Body) 47 | for scanner.Scan() { 48 | line := scanner.Text() 49 | if line == "" { 50 | continue 51 | } 52 | subdomains := session.Extractor.Extract(line) 53 | for _, subdomain := range subdomains { 54 | results <- subscraping.Result{ 55 | Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."), 56 | } 57 | s.results++ 58 | } 59 | } 60 | }() 61 | 62 | return results 63 | } 64 | 65 | // Name returns the name of the source 66 | func (s *Source) Name() string { 67 | return "digitorus" 68 | } 69 | 70 | func (s *Source) IsDefault() bool { 71 | return true 72 | } 73 | 74 | func (s *Source) HasRecursiveSupport() bool { 75 | return true 76 | } 77 | 78 | func (s *Source) NeedsKey() bool { 79 | return false 80 | } 81 | 82 | func (s *Source) AddApiKeys(_ []string) { 83 | // no key needed 84 | } 85 | 86 | func (s *Source) Statistics() subscraping.Statistics { 87 | return subscraping.Statistics{ 88 | Errors: s.errors, 89 | Results: s.results, 90 | TimeTaken: s.timeTaken, 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go: -------------------------------------------------------------------------------- 1 | // Package dnsdumpster logic 2 | package dnsdumpster 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | type response struct { 14 | A []struct { 15 | Host string `json:"host"` 16 | } `json:"a"` 17 | Ns []struct { 18 | Host string `json:"host"` 19 | } `json:"ns"` 20 | } 21 | 22 | // Source is the passive scraping agent 23 | type Source struct { 24 | apiKeys []string 25 | timeTaken time.Duration 26 | errors int 27 | results int 28 | skipped bool 29 | } 30 | 31 | // Run function returns all subdomains found with the service 32 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 33 | results := make(chan subscraping.Result) 34 | s.errors = 0 35 | s.results = 0 36 | 37 | go func() { 38 | defer func(startTime time.Time) { 39 | s.timeTaken = time.Since(startTime) 40 | close(results) 41 | }(time.Now()) 42 | 43 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 44 | if randomApiKey == "" { 45 | s.skipped = true 46 | return 47 | } 48 | 49 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) 50 | if err != nil { 51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 52 | s.errors++ 53 | session.DiscardHTTPResponse(resp) 54 | return 55 | } 56 | defer resp.Body.Close() 57 | 58 | var response response 59 | err = json.NewDecoder(resp.Body).Decode(&response) 60 | if err != nil { 61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 62 | s.errors++ 63 | resp.Body.Close() 64 | return 65 | } 66 | 67 | for _, record := range append(response.A, response.Ns...) { 68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host} 69 | s.results++ 70 | } 71 | 72 | }() 73 | 74 | return results 75 | } 76 | 77 | // Name returns the name of the source 78 | func (s *Source) Name() string { 79 | return "dnsdumpster" 80 | } 81 | 82 | func (s *Source) IsDefault() bool { 83 | return true 84 | } 85 | 86 | func (s *Source) HasRecursiveSupport() bool { 87 | return false 88 | } 89 | 90 | func (s *Source) NeedsKey() bool { 91 | return true 92 | } 93 | 94 | func (s *Source) AddApiKeys(keys []string) { 95 | s.apiKeys = keys 96 | } 97 | 98 | func (s *Source) Statistics() subscraping.Statistics { 99 | return subscraping.Statistics{ 100 | Errors: s.errors, 101 | Results: s.results, 102 | TimeTaken: s.timeTaken, 103 | Skipped: s.skipped, 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go: -------------------------------------------------------------------------------- 1 | package dnsrepo 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "strings" 9 | "time" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | // Source is the passive scraping agent 15 | type Source struct { 16 | apiKeys []string 17 | timeTaken time.Duration 18 | errors int 19 | results int 20 | skipped bool 21 | } 22 | 23 | type DnsRepoResponse []struct { 24 | Domain string `json:"domain"` 25 | } 26 | 27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 28 | results := make(chan subscraping.Result) 29 | s.errors = 0 30 | s.results = 0 31 | 32 | go func() { 33 | defer func(startTime time.Time) { 34 | s.timeTaken = time.Since(startTime) 35 | close(results) 36 | }(time.Now()) 37 | 38 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 39 | if randomApiKey == "" { 40 | s.skipped = true 41 | return 42 | } 43 | 44 | randomApiInfo := strings.Split(randomApiKey, ":") 45 | if len(randomApiInfo) != 2 { 46 | s.skipped = true 47 | return 48 | } 49 | 50 | token := randomApiInfo[0] 51 | apiKey := randomApiInfo[1] 52 | 53 | resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) 54 | if err != nil { 55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 56 | s.errors++ 57 | session.DiscardHTTPResponse(resp) 58 | return 59 | } 60 | responseData, err := io.ReadAll(resp.Body) 61 | if err != nil { 62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 63 | s.errors++ 64 | session.DiscardHTTPResponse(resp) 65 | return 66 | } 67 | resp.Body.Close() 68 | var result DnsRepoResponse 69 | err = json.Unmarshal(responseData, &result) 70 | if err != nil { 71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 72 | s.errors++ 73 | session.DiscardHTTPResponse(resp) 74 | return 75 | } 76 | for _, sub := range result { 77 | results <- subscraping.Result{ 78 | Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, "."), 79 | } 80 | s.results++ 81 | } 82 | 83 | }() 84 | 85 | return results 86 | } 87 | 88 | // Name returns the name of the source 89 | func (s *Source) Name() string { 90 | return "dnsrepo" 91 | } 92 | 93 | func (s *Source) IsDefault() bool { 94 | return true 95 | } 96 | 97 | func (s *Source) HasRecursiveSupport() bool { 98 | return false 99 | } 100 | 101 | func (s *Source) NeedsKey() bool { 102 | return true 103 | } 104 | 105 | func (s *Source) AddApiKeys(keys []string) { 106 | s.apiKeys = keys 107 | } 108 | 109 | func (s *Source) Statistics() subscraping.Statistics { 110 | return subscraping.Statistics{ 111 | Errors: s.errors, 112 | Results: s.results, 113 | TimeTaken: s.timeTaken, 114 | Skipped: s.skipped, 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/facebook/ctlogs_test.go: -------------------------------------------------------------------------------- 1 | package facebook 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/projectdiscovery/retryablehttp-go" 12 | "github.com/projectdiscovery/utils/generic" 13 | ) 14 | 15 | var ( 16 | fb_API_ID = "$FB_APP_ID" 17 | fb_API_SECRET = "$FB_APP_SECRET" 18 | ) 19 | 20 | func TestFacebookSource(t *testing.T) { 21 | if testing.Short() { 22 | t.Skip("skipping test in short mode.") 23 | } 24 | 25 | updateWithEnv(&fb_API_ID) 26 | updateWithEnv(&fb_API_SECRET) 27 | if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) { 28 | t.SkipNow() 29 | } 30 | k := apiKey{ 31 | AppID: fb_API_ID, 32 | Secret: fb_API_SECRET, 33 | } 34 | k.FetchAccessToken() 35 | if k.Error != nil { 36 | t.Fatal(k.Error) 37 | } 38 | 39 | fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken) 40 | resp, err := retryablehttp.Get(fetchURL) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | defer resp.Body.Close() 45 | bin, err := io.ReadAll(resp.Body) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | response := &response{} 50 | if err := json.Unmarshal(bin, response); err != nil { 51 | t.Fatal(err) 52 | } 53 | if len(response.Data) == 0 { 54 | t.Fatal("no data found") 55 | } 56 | } 57 | 58 | func updateWithEnv(key *string) { 59 | if key == nil { 60 | return 61 | } 62 | value := *key 63 | if strings.HasPrefix(value, "$") { 64 | *key = os.Getenv(value[1:]) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/facebook/types.go: -------------------------------------------------------------------------------- 1 | package facebook 2 | 3 | type authResponse struct { 4 | AccessToken string `json:"access_token"` 5 | } 6 | 7 | /* 8 | { 9 | "data": [ 10 | { 11 | "domains": [ 12 | "docs.hackerone.com" 13 | ], 14 | "id": "10056051421102939" 15 | }, 16 | ... 17 | ], 18 | "paging": { 19 | "cursors": { 20 | "before": "MTAwNTYwNTE0MjExMDI5MzkZD", 21 | "after": "Njc0OTczNTA5NTA1MzUxNwZDZD" 22 | }, 23 | "next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD" 24 | } 25 | } 26 | */ 27 | // example response 28 | 29 | type response struct { 30 | Data []struct { 31 | Domains []string `json:"domains"` 32 | } `json:"data"` 33 | Paging struct { 34 | Next string `json:"next"` 35 | } `json:"paging"` 36 | } 37 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/fofa/fofa.go: -------------------------------------------------------------------------------- 1 | // Package fofa logic 2 | package fofa 3 | 4 | import ( 5 | "context" 6 | "encoding/base64" 7 | "fmt" 8 | "regexp" 9 | "strings" 10 | "time" 11 | 12 | jsoniter "github.com/json-iterator/go" 13 | 14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 15 | ) 16 | 17 | type fofaResponse struct { 18 | Error bool `json:"error"` 19 | ErrMsg string `json:"errmsg"` 20 | Size int `json:"size"` 21 | Results []string `json:"results"` 22 | } 23 | 24 | // Source is the passive scraping agent 25 | type Source struct { 26 | apiKeys []apiKey 27 | timeTaken time.Duration 28 | errors int 29 | results int 30 | skipped bool 31 | } 32 | 33 | type apiKey struct { 34 | username string 35 | secret string 36 | } 37 | 38 | // Run function returns all subdomains found with the service 39 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 40 | results := make(chan subscraping.Result) 41 | s.errors = 0 42 | s.results = 0 43 | 44 | go func() { 45 | defer func(startTime time.Time) { 46 | s.timeTaken = time.Since(startTime) 47 | close(results) 48 | }(time.Now()) 49 | 50 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 51 | if randomApiKey.username == "" || randomApiKey.secret == "" { 52 | s.skipped = true 53 | return 54 | } 55 | 56 | // fofa api doc https://fofa.info/static_pages/api_help 57 | qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) 58 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) 59 | if err != nil && resp == nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | session.DiscardHTTPResponse(resp) 63 | return 64 | } 65 | 66 | var response fofaResponse 67 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 68 | if err != nil { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 70 | s.errors++ 71 | resp.Body.Close() 72 | return 73 | } 74 | resp.Body.Close() 75 | 76 | if response.Error { 77 | results <- subscraping.Result{ 78 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg), 79 | } 80 | s.errors++ 81 | return 82 | } 83 | 84 | if response.Size > 0 { 85 | for _, subdomain := range response.Results { 86 | if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { 87 | subdomain = subdomain[strings.Index(subdomain, "//")+2:] 88 | } 89 | re := regexp.MustCompile(`:\d+$`) 90 | if re.MatchString(subdomain) { 91 | subdomain = re.ReplaceAllString(subdomain, "") 92 | } 93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 94 | s.results++ 95 | } 96 | } 97 | }() 98 | 99 | return results 100 | } 101 | 102 | // Name returns the name of the source 103 | func (s *Source) Name() string { 104 | return "fofa" 105 | } 106 | 107 | func (s *Source) IsDefault() bool { 108 | return true 109 | } 110 | 111 | func (s *Source) HasRecursiveSupport() bool { 112 | return false 113 | } 114 | 115 | func (s *Source) NeedsKey() bool { 116 | return true 117 | } 118 | 119 | func (s *Source) AddApiKeys(keys []string) { 120 | s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { 121 | return apiKey{k, v} 122 | }) 123 | } 124 | 125 | func (s *Source) Statistics() subscraping.Statistics { 126 | return subscraping.Statistics{ 127 | Errors: s.errors, 128 | Results: s.results, 129 | TimeTaken: s.timeTaken, 130 | Skipped: s.skipped, 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/fullhunt/fullhunt.go: -------------------------------------------------------------------------------- 1 | package fullhunt 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | // fullhunt response 14 | type fullHuntResponse struct { 15 | Hosts []string `json:"hosts"` 16 | Message string `json:"message"` 17 | Status int `json:"status"` 18 | } 19 | 20 | // Source is the passive scraping agent 21 | type Source struct { 22 | apiKeys []string 23 | timeTaken time.Duration 24 | errors int 25 | results int 26 | skipped bool 27 | } 28 | 29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 30 | results := make(chan subscraping.Result) 31 | s.errors = 0 32 | s.results = 0 33 | 34 | go func() { 35 | defer func(startTime time.Time) { 36 | s.timeTaken = time.Since(startTime) 37 | close(results) 38 | }(time.Now()) 39 | 40 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 41 | if randomApiKey == "" { 42 | s.skipped = true 43 | return 44 | } 45 | 46 | resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) 47 | if err != nil { 48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 49 | s.errors++ 50 | session.DiscardHTTPResponse(resp) 51 | return 52 | } 53 | 54 | var response fullHuntResponse 55 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 56 | if err != nil { 57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 58 | s.errors++ 59 | resp.Body.Close() 60 | return 61 | } 62 | resp.Body.Close() 63 | for _, record := range response.Hosts { 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} 65 | s.results++ 66 | } 67 | }() 68 | 69 | return results 70 | } 71 | 72 | // Name returns the name of the source 73 | func (s *Source) Name() string { 74 | return "fullhunt" 75 | } 76 | 77 | func (s *Source) IsDefault() bool { 78 | return true 79 | } 80 | 81 | func (s *Source) HasRecursiveSupport() bool { 82 | return false 83 | } 84 | 85 | func (s *Source) NeedsKey() bool { 86 | return true 87 | } 88 | 89 | func (s *Source) AddApiKeys(keys []string) { 90 | s.apiKeys = keys 91 | } 92 | 93 | func (s *Source) Statistics() subscraping.Statistics { 94 | return subscraping.Statistics{ 95 | Errors: s.errors, 96 | Results: s.results, 97 | TimeTaken: s.timeTaken, 98 | Skipped: s.skipped, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/github/tokenmanager.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import "time" 4 | 5 | // Token struct 6 | type Token struct { 7 | Hash string 8 | RetryAfter int64 9 | ExceededTime time.Time 10 | } 11 | 12 | // Tokens is the internal struct to manage the current token 13 | // and the pool 14 | type Tokens struct { 15 | current int 16 | pool []Token 17 | } 18 | 19 | // NewTokenManager initialize the tokens pool 20 | func NewTokenManager(keys []string) *Tokens { 21 | pool := []Token{} 22 | for _, key := range keys { 23 | t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} 24 | pool = append(pool, t) 25 | } 26 | 27 | return &Tokens{ 28 | current: 0, 29 | pool: pool, 30 | } 31 | } 32 | 33 | func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { 34 | if r.current >= len(r.pool) { 35 | r.current %= len(r.pool) 36 | } 37 | if r.pool[r.current].RetryAfter == 0 { 38 | r.pool[r.current].ExceededTime = time.Now() 39 | r.pool[r.current].RetryAfter = retryAfter 40 | } 41 | } 42 | 43 | // Get returns a new token from the token pool 44 | func (r *Tokens) Get() *Token { 45 | resetExceededTokens(r) 46 | 47 | if r.current >= len(r.pool) { 48 | r.current %= len(r.pool) 49 | } 50 | 51 | result := &r.pool[r.current] 52 | r.current++ 53 | 54 | return result 55 | } 56 | 57 | func resetExceededTokens(r *Tokens) { 58 | for i, token := range r.pool { 59 | if token.RetryAfter > 0 { 60 | if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { 61 | r.pool[i].ExceededTime = time.Time{} 62 | r.pool[i].RetryAfter = 0 63 | } 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/gitlab/gitlab.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "regexp" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | jsoniter "github.com/json-iterator/go" 15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 16 | "github.com/tomnomnom/linkheader" 17 | ) 18 | 19 | // Source is the passive scraping agent 20 | type Source struct { 21 | apiKeys []string 22 | timeTaken time.Duration 23 | errors int 24 | results int 25 | skipped bool 26 | } 27 | 28 | type item struct { 29 | Data string `json:"data"` 30 | ProjectId int `json:"project_id"` 31 | Path string `json:"path"` 32 | Ref string `json:"ref"` 33 | } 34 | 35 | // Run function returns all subdomains found with the service 36 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 37 | results := make(chan subscraping.Result) 38 | s.errors = 0 39 | s.results = 0 40 | 41 | go func() { 42 | defer func(startTime time.Time) { 43 | s.timeTaken = time.Since(startTime) 44 | close(results) 45 | }(time.Now()) 46 | 47 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 48 | if randomApiKey == "" { 49 | return 50 | } 51 | 52 | headers := map[string]string{"PRIVATE-TOKEN": randomApiKey} 53 | 54 | searchURL := fmt.Sprintf("https://gitlab.com/api/v4/search?scope=blobs&search=%s&per_page=100", domain) 55 | s.enumerate(ctx, searchURL, domainRegexp(domain), headers, session, results) 56 | 57 | }() 58 | 59 | return results 60 | } 61 | 62 | func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { 63 | select { 64 | case <-ctx.Done(): 65 | return 66 | default: 67 | } 68 | 69 | resp, err := session.Get(ctx, searchURL, "", headers) 70 | if err != nil && resp == nil { 71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 72 | s.errors++ 73 | session.DiscardHTTPResponse(resp) 74 | return 75 | } 76 | 77 | defer resp.Body.Close() 78 | 79 | var items []item 80 | err = jsoniter.NewDecoder(resp.Body).Decode(&items) 81 | if err != nil { 82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 83 | s.errors++ 84 | return 85 | } 86 | 87 | var wg sync.WaitGroup 88 | wg.Add(len(items)) 89 | 90 | for _, it := range items { 91 | go func(item item) { 92 | // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path 93 | fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) 94 | resp, err := session.Get(ctx, fileUrl, "", headers) 95 | if err != nil { 96 | if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { 97 | session.DiscardHTTPResponse(resp) 98 | 99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 100 | s.errors++ 101 | return 102 | } 103 | } 104 | 105 | if resp.StatusCode == http.StatusOK { 106 | scanner := bufio.NewScanner(resp.Body) 107 | for scanner.Scan() { 108 | line := scanner.Text() 109 | if line == "" { 110 | continue 111 | } 112 | for _, subdomain := range domainRegexp.FindAllString(line, -1) { 113 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 114 | s.results++ 115 | } 116 | } 117 | resp.Body.Close() 118 | } 119 | defer wg.Done() 120 | }(it) 121 | } 122 | 123 | // Links header, first, next, last... 124 | linksHeader := linkheader.Parse(resp.Header.Get("Link")) 125 | // Process the next link recursively 126 | for _, link := range linksHeader { 127 | if link.Rel == "next" { 128 | nextURL, err := url.QueryUnescape(link.URL) 129 | if err != nil { 130 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 131 | s.errors++ 132 | return 133 | } 134 | 135 | s.enumerate(ctx, nextURL, domainRegexp, headers, session, results) 136 | } 137 | } 138 | 139 | wg.Wait() 140 | } 141 | 142 | func domainRegexp(domain string) *regexp.Regexp { 143 | rdomain := strings.ReplaceAll(domain, ".", "\\.") 144 | return regexp.MustCompile("(\\w[a-zA-Z0-9][a-zA-Z0-9-\\.]*)" + rdomain) 145 | } 146 | 147 | // Name returns the name of the source 148 | func (s *Source) Name() string { 149 | return "gitlab" 150 | } 151 | 152 | func (s *Source) IsDefault() bool { 153 | return false 154 | } 155 | 156 | func (s *Source) HasRecursiveSupport() bool { 157 | return false 158 | } 159 | 160 | func (s *Source) NeedsKey() bool { 161 | return true 162 | } 163 | 164 | func (s *Source) AddApiKeys(keys []string) { 165 | s.apiKeys = keys 166 | } 167 | 168 | // Statistics returns the statistics for the source 169 | func (s *Source) Statistics() subscraping.Statistics { 170 | return subscraping.Statistics{ 171 | Errors: s.errors, 172 | Results: s.results, 173 | TimeTaken: s.timeTaken, 174 | Skipped: s.skipped, 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/hackertarget/hackertarget.go: -------------------------------------------------------------------------------- 1 | // Package hackertarget logic 2 | package hackertarget 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct { 15 | timeTaken time.Duration 16 | errors int 17 | results int 18 | } 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | s.errors = 0 24 | s.results = 0 25 | 26 | go func() { 27 | defer func(startTime time.Time) { 28 | s.timeTaken = time.Since(startTime) 29 | close(results) 30 | }(time.Now()) 31 | 32 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain)) 33 | if err != nil { 34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 35 | s.errors++ 36 | session.DiscardHTTPResponse(resp) 37 | return 38 | } 39 | 40 | defer resp.Body.Close() 41 | 42 | scanner := bufio.NewScanner(resp.Body) 43 | for scanner.Scan() { 44 | line := scanner.Text() 45 | if line == "" { 46 | continue 47 | } 48 | match := session.Extractor.Extract(line) 49 | for _, subdomain := range match { 50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 51 | s.results++ 52 | } 53 | } 54 | }() 55 | 56 | return results 57 | } 58 | 59 | // Name returns the name of the source 60 | func (s *Source) Name() string { 61 | return "hackertarget" 62 | } 63 | 64 | func (s *Source) IsDefault() bool { 65 | return true 66 | } 67 | 68 | func (s *Source) HasRecursiveSupport() bool { 69 | return true 70 | } 71 | 72 | func (s *Source) NeedsKey() bool { 73 | return false 74 | } 75 | 76 | func (s *Source) AddApiKeys(_ []string) { 77 | // no key needed 78 | } 79 | 80 | func (s *Source) Statistics() subscraping.Statistics { 81 | return subscraping.Statistics{ 82 | Errors: s.errors, 83 | Results: s.results, 84 | TimeTaken: s.timeTaken, 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go: -------------------------------------------------------------------------------- 1 | // Package hudsonrock logic 2 | package hudsonrock 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | type hudsonrockResponse struct { 14 | Data struct { 15 | EmployeesUrls []struct { 16 | URL string `json:"url"` 17 | } `json:"employees_urls"` 18 | ClientsUrls []struct { 19 | URL string `json:"url"` 20 | } `json:"clients_urls"` 21 | } `json:"data"` 22 | } 23 | 24 | // Source is the passive scraping agent 25 | type Source struct { 26 | timeTaken time.Duration 27 | errors int 28 | results int 29 | } 30 | 31 | // Run function returns all subdomains found with the service 32 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 33 | results := make(chan subscraping.Result) 34 | s.errors = 0 35 | s.results = 0 36 | 37 | go func() { 38 | defer func(startTime time.Time) { 39 | s.timeTaken = time.Since(startTime) 40 | close(results) 41 | }(time.Now()) 42 | 43 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain)) 44 | if err != nil { 45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 46 | s.errors++ 47 | session.DiscardHTTPResponse(resp) 48 | return 49 | } 50 | defer resp.Body.Close() 51 | 52 | var response hudsonrockResponse 53 | err = json.NewDecoder(resp.Body).Decode(&response) 54 | if err != nil { 55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 56 | s.errors++ 57 | resp.Body.Close() 58 | return 59 | } 60 | 61 | for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) { 62 | for _, subdomain := range session.Extractor.Extract(record.URL) { 63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 64 | s.results++ 65 | } 66 | } 67 | 68 | }() 69 | 70 | return results 71 | } 72 | 73 | // Name returns the name of the source 74 | func (s *Source) Name() string { 75 | return "hudsonrock" 76 | } 77 | 78 | func (s *Source) IsDefault() bool { 79 | return false 80 | } 81 | 82 | func (s *Source) HasRecursiveSupport() bool { 83 | return false 84 | } 85 | 86 | func (s *Source) NeedsKey() bool { 87 | return false 88 | } 89 | 90 | func (s *Source) AddApiKeys(_ []string) { 91 | // no key needed 92 | } 93 | 94 | func (s *Source) Statistics() subscraping.Statistics { 95 | return subscraping.Statistics{ 96 | Errors: s.errors, 97 | Results: s.results, 98 | TimeTaken: s.timeTaken, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/hunter/hunter.go: -------------------------------------------------------------------------------- 1 | package hunter 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | type hunterResp struct { 14 | Code int `json:"code"` 15 | Data hunterData `json:"data"` 16 | Message string `json:"message"` 17 | } 18 | 19 | type infoArr struct { 20 | URL string `json:"url"` 21 | IP string `json:"ip"` 22 | Port int `json:"port"` 23 | Domain string `json:"domain"` 24 | Protocol string `json:"protocol"` 25 | } 26 | 27 | type hunterData struct { 28 | InfoArr []infoArr `json:"arr"` 29 | Total int `json:"total"` 30 | } 31 | 32 | // Source is the passive scraping agent 33 | type Source struct { 34 | apiKeys []string 35 | timeTaken time.Duration 36 | errors int 37 | results int 38 | skipped bool 39 | } 40 | 41 | // Run function returns all subdomains found with the service 42 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 43 | results := make(chan subscraping.Result) 44 | s.errors = 0 45 | s.results = 0 46 | 47 | go func() { 48 | defer func(startTime time.Time) { 49 | s.timeTaken = time.Since(startTime) 50 | close(results) 51 | }(time.Now()) 52 | 53 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 54 | if randomApiKey == "" { 55 | s.skipped = true 56 | return 57 | } 58 | 59 | var pages = 1 60 | for currentPage := 1; currentPage <= pages; currentPage++ { 61 | // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 62 | qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) 63 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) 64 | if err != nil && resp == nil { 65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 66 | s.errors++ 67 | session.DiscardHTTPResponse(resp) 68 | return 69 | } 70 | 71 | var response hunterResp 72 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 73 | if err != nil { 74 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 75 | s.errors++ 76 | resp.Body.Close() 77 | return 78 | } 79 | resp.Body.Close() 80 | 81 | if response.Code == 401 || response.Code == 400 { 82 | results <- subscraping.Result{ 83 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), 84 | } 85 | s.errors++ 86 | return 87 | } 88 | 89 | if response.Data.Total > 0 { 90 | for _, hunterInfo := range response.Data.InfoArr { 91 | subdomain := hunterInfo.Domain 92 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 93 | s.results++ 94 | } 95 | } 96 | pages = int(response.Data.Total/1000) + 1 97 | } 98 | }() 99 | 100 | return results 101 | } 102 | 103 | // Name returns the name of the source 104 | func (s *Source) Name() string { 105 | return "hunter" 106 | } 107 | 108 | func (s *Source) IsDefault() bool { 109 | return true 110 | } 111 | 112 | func (s *Source) HasRecursiveSupport() bool { 113 | return false 114 | } 115 | 116 | func (s *Source) NeedsKey() bool { 117 | return true 118 | } 119 | 120 | func (s *Source) AddApiKeys(keys []string) { 121 | s.apiKeys = keys 122 | } 123 | 124 | func (s *Source) Statistics() subscraping.Statistics { 125 | return subscraping.Statistics{ 126 | Errors: s.errors, 127 | Results: s.results, 128 | TimeTaken: s.timeTaken, 129 | Skipped: s.skipped, 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/intelx/intelx.go: -------------------------------------------------------------------------------- 1 | // Package intelx logic 2 | package intelx 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "time" 11 | 12 | jsoniter "github.com/json-iterator/go" 13 | 14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 15 | ) 16 | 17 | type searchResponseType struct { 18 | ID string `json:"id"` 19 | Status int `json:"status"` 20 | } 21 | 22 | type selectorType struct { 23 | Selectvalue string `json:"selectorvalue"` 24 | } 25 | 26 | type searchResultType struct { 27 | Selectors []selectorType `json:"selectors"` 28 | Status int `json:"status"` 29 | } 30 | 31 | type requestBody struct { 32 | Term string 33 | Maxresults int 34 | Media int 35 | Target int 36 | Terminate []int 37 | Timeout int 38 | } 39 | 40 | // Source is the passive scraping agent 41 | type Source struct { 42 | apiKeys []apiKey 43 | timeTaken time.Duration 44 | errors int 45 | results int 46 | skipped bool 47 | } 48 | 49 | type apiKey struct { 50 | host string 51 | key string 52 | } 53 | 54 | // Run function returns all subdomains found with the service 55 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 56 | results := make(chan subscraping.Result) 57 | s.errors = 0 58 | s.results = 0 59 | 60 | go func() { 61 | defer func(startTime time.Time) { 62 | s.timeTaken = time.Since(startTime) 63 | close(results) 64 | }(time.Now()) 65 | 66 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 67 | if randomApiKey.host == "" || randomApiKey.key == "" { 68 | s.skipped = true 69 | return 70 | } 71 | 72 | searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", randomApiKey.host, randomApiKey.key) 73 | reqBody := requestBody{ 74 | Term: domain, 75 | Maxresults: 100000, 76 | Media: 0, 77 | Target: 1, 78 | Timeout: 20, 79 | } 80 | 81 | body, err := json.Marshal(reqBody) 82 | if err != nil { 83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 84 | s.errors++ 85 | return 86 | } 87 | 88 | resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) 89 | if err != nil { 90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 91 | s.errors++ 92 | session.DiscardHTTPResponse(resp) 93 | return 94 | } 95 | 96 | var response searchResponseType 97 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 98 | if err != nil { 99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 100 | s.errors++ 101 | resp.Body.Close() 102 | return 103 | } 104 | 105 | resp.Body.Close() 106 | 107 | resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) 108 | status := 0 109 | for status == 0 || status == 3 { 110 | resp, err = session.Get(ctx, resultsURL, "", nil) 111 | if err != nil { 112 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 113 | s.errors++ 114 | session.DiscardHTTPResponse(resp) 115 | return 116 | } 117 | var response searchResultType 118 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 119 | if err != nil { 120 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 121 | s.errors++ 122 | resp.Body.Close() 123 | return 124 | } 125 | 126 | _, err = io.ReadAll(resp.Body) 127 | if err != nil { 128 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 129 | s.errors++ 130 | resp.Body.Close() 131 | return 132 | } 133 | resp.Body.Close() 134 | 135 | status = response.Status 136 | for _, hostname := range response.Selectors { 137 | results <- subscraping.Result{ 138 | Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue, 139 | } 140 | s.results++ 141 | } 142 | } 143 | }() 144 | 145 | return results 146 | } 147 | 148 | // Name returns the name of the source 149 | func (s *Source) Name() string { 150 | return "intelx" 151 | } 152 | 153 | func (s *Source) IsDefault() bool { 154 | return true 155 | } 156 | 157 | func (s *Source) HasRecursiveSupport() bool { 158 | return false 159 | } 160 | 161 | func (s *Source) NeedsKey() bool { 162 | return true 163 | } 164 | 165 | func (s *Source) AddApiKeys(keys []string) { 166 | s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { 167 | return apiKey{k, v} 168 | }) 169 | } 170 | 171 | func (s *Source) Statistics() subscraping.Statistics { 172 | return subscraping.Statistics{ 173 | Errors: s.errors, 174 | Results: s.results, 175 | TimeTaken: s.timeTaken, 176 | Skipped: s.skipped, 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/leakix/leakix.go: -------------------------------------------------------------------------------- 1 | // Package leakix logic 2 | package leakix 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct { 15 | apiKeys []string 16 | timeTaken time.Duration 17 | errors int 18 | results int 19 | skipped bool 20 | } 21 | 22 | // Run function returns all subdomains found with the service 23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 24 | results := make(chan subscraping.Result) 25 | s.errors = 0 26 | s.results = 0 27 | 28 | go func() { 29 | defer func(startTime time.Time) { 30 | s.timeTaken = time.Since(startTime) 31 | close(results) 32 | }(time.Now()) 33 | // Default headers 34 | headers := map[string]string{ 35 | "accept": "application/json", 36 | } 37 | // Pick an API key 38 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 39 | if randomApiKey != "" { 40 | headers["api-key"] = randomApiKey 41 | } 42 | // Request 43 | resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers) 44 | if err != nil { 45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 46 | s.errors++ 47 | return 48 | } 49 | defer resp.Body.Close() 50 | if resp.StatusCode != 200 { 51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} 52 | s.errors++ 53 | return 54 | } 55 | // Parse and return results 56 | var subdomains []subResponse 57 | decoder := json.NewDecoder(resp.Body) 58 | err = decoder.Decode(&subdomains) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | return 63 | } 64 | for _, result := range subdomains { 65 | results <- subscraping.Result{ 66 | Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain, 67 | } 68 | s.results++ 69 | } 70 | }() 71 | return results 72 | } 73 | 74 | // Name returns the name of the source 75 | func (s *Source) Name() string { 76 | return "leakix" 77 | } 78 | 79 | func (s *Source) IsDefault() bool { 80 | return true 81 | } 82 | 83 | func (s *Source) HasRecursiveSupport() bool { 84 | return true 85 | } 86 | 87 | func (s *Source) NeedsKey() bool { 88 | return true 89 | } 90 | 91 | func (s *Source) AddApiKeys(keys []string) { 92 | s.apiKeys = keys 93 | } 94 | 95 | func (s *Source) Statistics() subscraping.Statistics { 96 | return subscraping.Statistics{ 97 | Errors: s.errors, 98 | Results: s.results, 99 | TimeTaken: s.timeTaken, 100 | Skipped: s.skipped, 101 | } 102 | } 103 | 104 | type subResponse struct { 105 | Subdomain string `json:"subdomain"` 106 | DistinctIps int `json:"distinct_ips"` 107 | LastSeen time.Time `json:"last_seen"` 108 | } 109 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/netlas/netlas.go: -------------------------------------------------------------------------------- 1 | // Package netlas logic 2 | package netlas 3 | 4 | import ( 5 | "context" 6 | "io" 7 | "strings" 8 | 9 | "encoding/json" 10 | "fmt" 11 | "net/http" 12 | "net/url" 13 | "time" 14 | 15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 16 | ) 17 | 18 | type Item struct { 19 | Data struct { 20 | A []string `json:"a,omitempty"` 21 | Txt []string `json:"txt,omitempty"` 22 | LastUpdated string `json:"last_updated,omitempty"` 23 | Timestamp string `json:"@timestamp,omitempty"` 24 | Ns []string `json:"ns,omitempty"` 25 | Level int `json:"level,omitempty"` 26 | Zone string `json:"zone,omitempty"` 27 | Domain string `json:"domain,omitempty"` 28 | Cname []string `json:"cname,omitempty"` 29 | Mx []string `json:"mx,omitempty"` 30 | } `json:"data"` 31 | } 32 | 33 | type DomainsCountResponse struct { 34 | Count int `json:"count"` 35 | } 36 | 37 | // Source is the passive scraping agent 38 | type Source struct { 39 | apiKeys []string 40 | timeTaken time.Duration 41 | errors int 42 | results int 43 | skipped bool 44 | } 45 | 46 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 47 | results := make(chan subscraping.Result) 48 | s.errors = 0 49 | s.results = 0 50 | 51 | go func() { 52 | defer func(startTime time.Time) { 53 | s.timeTaken = time.Since(startTime) 54 | close(results) 55 | }(time.Now()) 56 | 57 | // To get count of domains 58 | endpoint := "https://app.netlas.io/api/domains_count/" 59 | params := url.Values{} 60 | countQuery := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) 61 | params.Set("q", countQuery) 62 | countUrl := endpoint + "?" + params.Encode() 63 | 64 | // Pick an API key 65 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 66 | resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ 67 | "accept": "application/json", 68 | "X-API-Key": randomApiKey, 69 | }, nil, subscraping.BasicAuth{}) 70 | 71 | if err != nil { 72 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 73 | s.errors++ 74 | return 75 | } else if resp.StatusCode != 200 { 76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} 77 | s.errors++ 78 | return 79 | } 80 | defer resp.Body.Close() 81 | 82 | body, err := io.ReadAll(resp.Body) 83 | if err != nil { 84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} 85 | s.errors++ 86 | return 87 | } 88 | 89 | // Parse the JSON response 90 | var domainsCount DomainsCountResponse 91 | err = json.Unmarshal(body, &domainsCount) 92 | if err != nil { 93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 94 | s.errors++ 95 | return 96 | } 97 | 98 | // Make a single POST request to get all domains via download method 99 | 100 | apiUrl := "https://app.netlas.io/api/domains/download/" 101 | query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) 102 | requestBody := map[string]interface{}{ 103 | "q": query, 104 | "fields": []string{"*"}, 105 | "source_type": "include", 106 | "size": domainsCount.Count, 107 | } 108 | jsonRequestBody, err := json.Marshal(requestBody) 109 | if err != nil { 110 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error marshaling request body")} 111 | s.errors++ 112 | return 113 | } 114 | 115 | // Pick an API key 116 | randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) 117 | 118 | resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ 119 | "accept": "application/json", 120 | "X-API-Key": randomApiKey, 121 | "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) 122 | if err != nil { 123 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 124 | s.errors++ 125 | return 126 | } 127 | defer resp.Body.Close() 128 | body, err = io.ReadAll(resp.Body) 129 | if err != nil { 130 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} 131 | s.errors++ 132 | return 133 | } 134 | 135 | if resp.StatusCode == 429 { 136 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} 137 | s.errors++ 138 | return 139 | } 140 | 141 | // Parse the response body and extract the domain values 142 | var data []Item 143 | err = json.Unmarshal(body, &data) 144 | if err != nil { 145 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 146 | s.errors++ 147 | return 148 | } 149 | 150 | for _, item := range data { 151 | results <- subscraping.Result{ 152 | Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain, 153 | } 154 | s.results++ 155 | } 156 | 157 | }() 158 | 159 | return results 160 | } 161 | 162 | // Name returns the name of the source 163 | func (s *Source) Name() string { 164 | return "netlas" 165 | } 166 | 167 | func (s *Source) IsDefault() bool { 168 | return false 169 | } 170 | 171 | func (s *Source) HasRecursiveSupport() bool { 172 | return false 173 | } 174 | 175 | func (s *Source) NeedsKey() bool { 176 | return true 177 | } 178 | 179 | func (s *Source) AddApiKeys(keys []string) { 180 | s.apiKeys = keys 181 | } 182 | 183 | func (s *Source) Statistics() subscraping.Statistics { 184 | return subscraping.Statistics{ 185 | Errors: s.errors, 186 | Results: s.results, 187 | TimeTaken: s.timeTaken, 188 | Skipped: s.skipped, 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/quake/quake.go: -------------------------------------------------------------------------------- 1 | // Package quake logic 2 | package quake 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "strings" 9 | "time" 10 | 11 | jsoniter "github.com/json-iterator/go" 12 | 13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 14 | ) 15 | 16 | type quakeResults struct { 17 | Code int `json:"code"` 18 | Message string `json:"message"` 19 | Data []struct { 20 | Service struct { 21 | HTTP struct { 22 | Host string `json:"host"` 23 | } `json:"http"` 24 | } 25 | } `json:"data"` 26 | Meta struct { 27 | Pagination struct { 28 | Total int `json:"total"` 29 | } `json:"pagination"` 30 | } `json:"meta"` 31 | } 32 | 33 | // Source is the passive scraping agent 34 | type Source struct { 35 | apiKeys []string 36 | timeTaken time.Duration 37 | errors int 38 | results int 39 | skipped bool 40 | } 41 | 42 | // Run function returns all subdomains found with the service 43 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 44 | results := make(chan subscraping.Result) 45 | s.errors = 0 46 | s.results = 0 47 | 48 | go func() { 49 | defer func(startTime time.Time) { 50 | s.timeTaken = time.Since(startTime) 51 | close(results) 52 | }(time.Now()) 53 | 54 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 55 | if randomApiKey == "" { 56 | s.skipped = true 57 | return 58 | } 59 | 60 | // quake api doc https://quake.360.cn/quake/#/help 61 | var requestBody = []byte(fmt.Sprintf(`{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)) 62 | resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ 63 | "Content-Type": "application/json", "X-QuakeToken": randomApiKey, 64 | }, bytes.NewReader(requestBody)) 65 | if err != nil { 66 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 67 | s.errors++ 68 | session.DiscardHTTPResponse(resp) 69 | return 70 | } 71 | 72 | var response quakeResults 73 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 74 | if err != nil { 75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 76 | s.errors++ 77 | resp.Body.Close() 78 | return 79 | } 80 | resp.Body.Close() 81 | 82 | if response.Code != 0 { 83 | results <- subscraping.Result{ 84 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), 85 | } 86 | s.errors++ 87 | return 88 | } 89 | 90 | if response.Meta.Pagination.Total > 0 { 91 | for _, quakeDomain := range response.Data { 92 | subdomain := quakeDomain.Service.HTTP.Host 93 | if strings.ContainsAny(subdomain, "暂无权限") { 94 | subdomain = "" 95 | } 96 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 97 | s.results++ 98 | } 99 | } 100 | }() 101 | 102 | return results 103 | } 104 | 105 | // Name returns the name of the source 106 | func (s *Source) Name() string { 107 | return "quake" 108 | } 109 | 110 | func (s *Source) IsDefault() bool { 111 | return true 112 | } 113 | 114 | func (s *Source) HasRecursiveSupport() bool { 115 | return false 116 | } 117 | 118 | func (s *Source) NeedsKey() bool { 119 | return true 120 | } 121 | 122 | func (s *Source) AddApiKeys(keys []string) { 123 | s.apiKeys = keys 124 | } 125 | 126 | func (s *Source) Statistics() subscraping.Statistics { 127 | return subscraping.Statistics{ 128 | Errors: s.errors, 129 | Results: s.results, 130 | TimeTaken: s.timeTaken, 131 | Skipped: s.skipped, 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/rapiddns/rapiddns.go: -------------------------------------------------------------------------------- 1 | // Package rapiddns is a RapidDNS Scraping Engine in Golang 2 | package rapiddns 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | "regexp" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | var pagePattern = regexp.MustCompile(`class="page-link" href="/subdomain/[^"]+\?page=(\d+)">`) 16 | 17 | // Source is the passive scraping agent 18 | type Source struct { 19 | timeTaken time.Duration 20 | errors int 21 | results int 22 | } 23 | 24 | // Run function returns all subdomains found with the service 25 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 26 | results := make(chan subscraping.Result) 27 | s.errors = 0 28 | s.results = 0 29 | 30 | go func() { 31 | defer func(startTime time.Time) { 32 | s.timeTaken = time.Since(startTime) 33 | close(results) 34 | }(time.Now()) 35 | 36 | page := 1 37 | maxPages := 1 38 | for { 39 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | s.errors++ 43 | session.DiscardHTTPResponse(resp) 44 | return 45 | } 46 | 47 | body, err := io.ReadAll(resp.Body) 48 | if err != nil { 49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 50 | s.errors++ 51 | resp.Body.Close() 52 | return 53 | } 54 | 55 | resp.Body.Close() 56 | 57 | src := string(body) 58 | for _, subdomain := range session.Extractor.Extract(src) { 59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 60 | s.results++ 61 | } 62 | 63 | if maxPages == 1 { 64 | matches := pagePattern.FindAllStringSubmatch(src, -1) 65 | if len(matches) > 0 { 66 | lastMatch := matches[len(matches)-1] 67 | if len(lastMatch) > 1 { 68 | maxPages, _ = strconv.Atoi(lastMatch[1]) 69 | } 70 | } 71 | } 72 | 73 | if page >= maxPages { 74 | break 75 | } 76 | page++ 77 | } 78 | }() 79 | 80 | return results 81 | } 82 | 83 | // Name returns the name of the source 84 | func (s *Source) Name() string { 85 | return "rapiddns" 86 | } 87 | 88 | func (s *Source) IsDefault() bool { 89 | return false 90 | } 91 | 92 | func (s *Source) HasRecursiveSupport() bool { 93 | return false 94 | } 95 | 96 | func (s *Source) NeedsKey() bool { 97 | return false 98 | } 99 | 100 | func (s *Source) AddApiKeys(_ []string) { 101 | // no key needed 102 | } 103 | 104 | func (s *Source) Statistics() subscraping.Statistics { 105 | return subscraping.Statistics{ 106 | Errors: s.errors, 107 | Results: s.results, 108 | TimeTaken: s.timeTaken, 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/reconcloud/reconcloud.go: -------------------------------------------------------------------------------- 1 | // Package reconcloud logic 2 | package reconcloud 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | type reconCloudResponse struct { 14 | MsgType string `json:"msg_type"` 15 | RequestID string `json:"request_id"` 16 | OnCache bool `json:"on_cache"` 17 | Step string `json:"step"` 18 | CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"` 19 | } 20 | 21 | type cloudAssetsList struct { 22 | Key string `json:"key"` 23 | Domain string `json:"domain"` 24 | CloudProvider string `json:"cloud_provider"` 25 | } 26 | 27 | // Source is the passive scraping agent 28 | type Source struct { 29 | timeTaken time.Duration 30 | errors int 31 | results int 32 | } 33 | 34 | // Run function returns all subdomains found with the service 35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 36 | results := make(chan subscraping.Result) 37 | s.errors = 0 38 | s.results = 0 39 | 40 | go func() { 41 | defer func(startTime time.Time) { 42 | s.timeTaken = time.Since(startTime) 43 | close(results) 44 | }(time.Now()) 45 | 46 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) 47 | if err != nil && resp == nil { 48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 49 | s.errors++ 50 | session.DiscardHTTPResponse(resp) 51 | return 52 | } 53 | 54 | var response reconCloudResponse 55 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 56 | if err != nil { 57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 58 | s.errors++ 59 | resp.Body.Close() 60 | return 61 | } 62 | resp.Body.Close() 63 | 64 | if len(response.CloudAssetsList) > 0 { 65 | for _, cloudAsset := range response.CloudAssetsList { 66 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} 67 | s.results++ 68 | } 69 | } 70 | }() 71 | 72 | return results 73 | } 74 | 75 | // Name returns the name of the source 76 | func (s *Source) Name() string { 77 | return "reconcloud" 78 | } 79 | 80 | func (s *Source) IsDefault() bool { 81 | return true 82 | } 83 | 84 | func (s *Source) HasRecursiveSupport() bool { 85 | return true 86 | } 87 | 88 | func (s *Source) NeedsKey() bool { 89 | return false 90 | } 91 | 92 | func (s *Source) AddApiKeys(_ []string) { 93 | // no key needed 94 | } 95 | 96 | func (s *Source) Statistics() subscraping.Statistics { 97 | return subscraping.Statistics{ 98 | Errors: s.errors, 99 | Results: s.results, 100 | TimeTaken: s.timeTaken, 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go: -------------------------------------------------------------------------------- 1 | // Package redhuntlabs logic 2 | package redhuntlabs 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | type Response struct { 16 | Subdomains []string `json:"subdomains"` 17 | Metadata ResponseMetadata `json:"metadata"` 18 | } 19 | 20 | type ResponseMetadata struct { 21 | ResultCount int `json:"result_count"` 22 | PageSize int `json:"page_size"` 23 | PageNumber int `json:"page_number"` 24 | } 25 | 26 | type Source struct { 27 | apiKeys []string 28 | timeTaken time.Duration 29 | errors int 30 | results int 31 | skipped bool 32 | } 33 | 34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 35 | results := make(chan subscraping.Result) 36 | s.errors = 0 37 | s.results = 0 38 | pageSize := 1000 39 | go func() { 40 | defer func(startTime time.Time) { 41 | s.timeTaken = time.Since(startTime) 42 | close(results) 43 | }(time.Now()) 44 | 45 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 46 | if randomApiKey == "" || !strings.Contains(randomApiKey, ":") { 47 | s.skipped = true 48 | return 49 | } 50 | 51 | randomApiInfo := strings.Split(randomApiKey, ":") 52 | if len(randomApiInfo) != 3 { 53 | s.skipped = true 54 | return 55 | } 56 | baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1] 57 | requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"} 58 | getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize) 59 | resp, err := session.Get(ctx, getUrl, "", requestHeaders) 60 | if err != nil { 61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} 62 | session.DiscardHTTPResponse(resp) 63 | s.errors++ 64 | return 65 | } 66 | var response Response 67 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 68 | if err != nil { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 70 | resp.Body.Close() 71 | s.errors++ 72 | return 73 | } 74 | 75 | resp.Body.Close() 76 | if response.Metadata.ResultCount > pageSize { 77 | totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize 78 | for page := 1; page <= totalPages; page++ { 79 | getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) 80 | resp, err := session.Get(ctx, getUrl, "", requestHeaders) 81 | if err != nil { 82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} 83 | session.DiscardHTTPResponse(resp) 84 | s.errors++ 85 | return 86 | } 87 | 88 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 89 | if err != nil { 90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 91 | resp.Body.Close() 92 | s.errors++ 93 | continue 94 | } 95 | 96 | resp.Body.Close() 97 | 98 | for _, subdomain := range response.Subdomains { 99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 100 | s.results++ 101 | } 102 | } 103 | } else { 104 | for _, subdomain := range response.Subdomains { 105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 106 | s.results++ 107 | } 108 | } 109 | 110 | }() 111 | return results 112 | } 113 | 114 | func (s *Source) Name() string { 115 | return "redhuntlabs" 116 | } 117 | 118 | func (s *Source) IsDefault() bool { 119 | return true 120 | } 121 | 122 | func (s *Source) HasRecursiveSupport() bool { 123 | return false 124 | } 125 | 126 | func (s *Source) NeedsKey() bool { 127 | return true 128 | } 129 | 130 | func (s *Source) AddApiKeys(keys []string) { 131 | s.apiKeys = keys 132 | } 133 | 134 | func (s *Source) Statistics() subscraping.Statistics { 135 | return subscraping.Statistics{ 136 | Errors: s.errors, 137 | Results: s.results, 138 | TimeTaken: s.timeTaken, 139 | Skipped: s.skipped, 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/riddler/riddler.go: -------------------------------------------------------------------------------- 1 | // Package riddler logic 2 | package riddler 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct { 15 | timeTaken time.Duration 16 | errors int 17 | results int 18 | } 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | s.errors = 0 24 | s.results = 0 25 | 26 | go func() { 27 | defer func(startTime time.Time) { 28 | s.timeTaken = time.Since(startTime) 29 | close(results) 30 | }(time.Now()) 31 | 32 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) 33 | if err != nil { 34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 35 | s.errors++ 36 | session.DiscardHTTPResponse(resp) 37 | return 38 | } 39 | 40 | scanner := bufio.NewScanner(resp.Body) 41 | for scanner.Scan() { 42 | line := scanner.Text() 43 | if line == "" { 44 | continue 45 | } 46 | for _, subdomain := range session.Extractor.Extract(line) { 47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 48 | s.results++ 49 | } 50 | } 51 | resp.Body.Close() 52 | }() 53 | 54 | return results 55 | } 56 | 57 | // Name returns the name of the source 58 | func (s *Source) Name() string { 59 | return "riddler" 60 | } 61 | 62 | func (s *Source) IsDefault() bool { 63 | return true 64 | } 65 | 66 | func (s *Source) HasRecursiveSupport() bool { 67 | return false 68 | } 69 | 70 | func (s *Source) NeedsKey() bool { 71 | return false 72 | } 73 | 74 | func (s *Source) AddApiKeys(_ []string) { 75 | // no key needed 76 | } 77 | 78 | func (s *Source) Statistics() subscraping.Statistics { 79 | return subscraping.Statistics{ 80 | Errors: s.errors, 81 | Results: s.results, 82 | TimeTaken: s.timeTaken, 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/robtex/robtext.go: -------------------------------------------------------------------------------- 1 | // Package robtex logic 2 | package robtex 3 | 4 | import ( 5 | "bufio" 6 | "bytes" 7 | "context" 8 | "fmt" 9 | "time" 10 | 11 | jsoniter "github.com/json-iterator/go" 12 | 13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 14 | ) 15 | 16 | const ( 17 | addrRecord = "A" 18 | iPv6AddrRecord = "AAAA" 19 | baseURL = "https://proapi.robtex.com/pdns" 20 | ) 21 | 22 | // Source is the passive scraping agent 23 | type Source struct { 24 | apiKeys []string 25 | timeTaken time.Duration 26 | errors int 27 | results int 28 | skipped bool 29 | } 30 | 31 | type result struct { 32 | Rrname string `json:"rrname"` 33 | Rrdata string `json:"rrdata"` 34 | Rrtype string `json:"rrtype"` 35 | } 36 | 37 | // Run function returns all subdomains found with the service 38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 39 | results := make(chan subscraping.Result) 40 | s.errors = 0 41 | s.results = 0 42 | 43 | go func() { 44 | defer func(startTime time.Time) { 45 | s.timeTaken = time.Since(startTime) 46 | close(results) 47 | }(time.Now()) 48 | 49 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 50 | if randomApiKey == "" { 51 | s.skipped = true 52 | return 53 | } 54 | 55 | headers := map[string]string{"Content-Type": "application/x-ndjson"} 56 | 57 | ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, randomApiKey), headers) 58 | if err != nil { 59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 60 | s.errors++ 61 | return 62 | } 63 | 64 | for _, result := range ips { 65 | if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { 66 | domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers) 67 | if err != nil { 68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 69 | s.errors++ 70 | return 71 | } 72 | for _, result := range domains { 73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} 74 | s.results++ 75 | } 76 | } 77 | } 78 | }() 79 | 80 | return results 81 | } 82 | 83 | func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) { 84 | var results []result 85 | 86 | resp, err := session.Get(ctx, targetURL, "", headers) 87 | if err != nil { 88 | session.DiscardHTTPResponse(resp) 89 | return results, err 90 | } 91 | 92 | scanner := bufio.NewScanner(resp.Body) 93 | for scanner.Scan() { 94 | line := scanner.Text() 95 | if line == "" { 96 | continue 97 | } 98 | var response result 99 | err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) 100 | if err != nil { 101 | return results, err 102 | } 103 | 104 | results = append(results, response) 105 | } 106 | 107 | resp.Body.Close() 108 | 109 | return results, nil 110 | } 111 | 112 | // Name returns the name of the source 113 | func (s *Source) Name() string { 114 | return "robtex" 115 | } 116 | 117 | func (s *Source) IsDefault() bool { 118 | return true 119 | } 120 | 121 | func (s *Source) HasRecursiveSupport() bool { 122 | return false 123 | } 124 | 125 | func (s *Source) NeedsKey() bool { 126 | return true 127 | } 128 | 129 | func (s *Source) AddApiKeys(keys []string) { 130 | s.apiKeys = keys 131 | } 132 | 133 | func (s *Source) Statistics() subscraping.Statistics { 134 | return subscraping.Statistics{ 135 | Errors: s.errors, 136 | Results: s.results, 137 | TimeTaken: s.timeTaken, 138 | Skipped: s.skipped, 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/securitytrails/securitytrails.go: -------------------------------------------------------------------------------- 1 | // Package securitytrails logic 2 | package securitytrails 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | jsoniter "github.com/json-iterator/go" 13 | 14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 15 | "github.com/projectdiscovery/utils/ptr" 16 | ) 17 | 18 | type response struct { 19 | Meta struct { 20 | ScrollID string `json:"scroll_id"` 21 | } `json:"meta"` 22 | Records []struct { 23 | Hostname string `json:"hostname"` 24 | } `json:"records"` 25 | Subdomains []string `json:"subdomains"` 26 | } 27 | 28 | // Source is the passive scraping agent 29 | type Source struct { 30 | apiKeys []string 31 | timeTaken time.Duration 32 | errors int 33 | results int 34 | skipped bool 35 | } 36 | 37 | // Run function returns all subdomains found with the service 38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 39 | results := make(chan subscraping.Result) 40 | s.errors = 0 41 | s.results = 0 42 | 43 | go func() { 44 | defer func(startTime time.Time) { 45 | s.timeTaken = time.Since(startTime) 46 | close(results) 47 | }(time.Now()) 48 | 49 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 50 | if randomApiKey == "" { 51 | s.skipped = true 52 | return 53 | } 54 | 55 | var scrollId string 56 | headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} 57 | 58 | for { 59 | var resp *http.Response 60 | var err error 61 | 62 | if scrollId == "" { 63 | var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain)) 64 | resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", 65 | headers, bytes.NewReader(requestBody)) 66 | } else { 67 | resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) 68 | } 69 | 70 | if err != nil && ptr.Safe(resp).StatusCode == 403 { 71 | resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) 72 | } 73 | 74 | if err != nil { 75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 76 | s.errors++ 77 | session.DiscardHTTPResponse(resp) 78 | return 79 | } 80 | 81 | var securityTrailsResponse response 82 | err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) 83 | if err != nil { 84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 85 | s.errors++ 86 | resp.Body.Close() 87 | return 88 | } 89 | 90 | resp.Body.Close() 91 | 92 | for _, record := range securityTrailsResponse.Records { 93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} 94 | s.results++ 95 | } 96 | 97 | for _, subdomain := range securityTrailsResponse.Subdomains { 98 | if strings.HasSuffix(subdomain, ".") { 99 | subdomain += domain 100 | } else { 101 | subdomain = subdomain + "." + domain 102 | } 103 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 104 | s.results++ 105 | } 106 | 107 | scrollId = securityTrailsResponse.Meta.ScrollID 108 | 109 | if scrollId == "" { 110 | break 111 | } 112 | } 113 | }() 114 | 115 | return results 116 | } 117 | 118 | // Name returns the name of the source 119 | func (s *Source) Name() string { 120 | return "securitytrails" 121 | } 122 | 123 | func (s *Source) IsDefault() bool { 124 | return true 125 | } 126 | 127 | func (s *Source) HasRecursiveSupport() bool { 128 | return true 129 | } 130 | 131 | func (s *Source) NeedsKey() bool { 132 | return true 133 | } 134 | 135 | func (s *Source) AddApiKeys(keys []string) { 136 | s.apiKeys = keys 137 | } 138 | 139 | func (s *Source) Statistics() subscraping.Statistics { 140 | return subscraping.Statistics{ 141 | Errors: s.errors, 142 | Results: s.results, 143 | TimeTaken: s.timeTaken, 144 | Skipped: s.skipped, 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/shodan/shodan.go: -------------------------------------------------------------------------------- 1 | // Package shodan logic 2 | package shodan 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | // Source is the passive scraping agent 15 | type Source struct { 16 | apiKeys []string 17 | timeTaken time.Duration 18 | errors int 19 | results int 20 | skipped bool 21 | } 22 | 23 | type dnsdbLookupResponse struct { 24 | Domain string `json:"domain"` 25 | Subdomains []string `json:"subdomains"` 26 | Result int `json:"result"` 27 | Error string `json:"error"` 28 | More bool `json:"more"` 29 | } 30 | 31 | // Run function returns all subdomains found with the service 32 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 33 | results := make(chan subscraping.Result) 34 | s.errors = 0 35 | s.results = 0 36 | 37 | go func() { 38 | defer func(startTime time.Time) { 39 | s.timeTaken = time.Since(startTime) 40 | close(results) 41 | }(time.Now()) 42 | 43 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 44 | if randomApiKey == "" { 45 | s.skipped = true 46 | return 47 | } 48 | 49 | page := 1 50 | for { 51 | 52 | searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) 53 | resp, err := session.SimpleGet(ctx, searchURL) 54 | if err != nil { 55 | session.DiscardHTTPResponse(resp) 56 | return 57 | } 58 | 59 | defer resp.Body.Close() 60 | 61 | var response dnsdbLookupResponse 62 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 63 | if err != nil { 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 65 | s.errors++ 66 | return 67 | } 68 | 69 | if response.Error != "" { 70 | results <- subscraping.Result{ 71 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), 72 | } 73 | s.errors++ 74 | return 75 | } 76 | 77 | for _, data := range response.Subdomains { 78 | value := fmt.Sprintf("%s.%s", data, response.Domain) 79 | results <- subscraping.Result{ 80 | Source: s.Name(), Type: subscraping.Subdomain, Value: value, 81 | } 82 | s.results++ 83 | } 84 | 85 | if !response.More { 86 | break 87 | } 88 | page++ 89 | } 90 | }() 91 | 92 | return results 93 | } 94 | 95 | // Name returns the name of the source 96 | func (s *Source) Name() string { 97 | return "shodan" 98 | } 99 | 100 | func (s *Source) IsDefault() bool { 101 | return true 102 | } 103 | 104 | func (s *Source) HasRecursiveSupport() bool { 105 | return false 106 | } 107 | 108 | func (s *Source) NeedsKey() bool { 109 | return true 110 | } 111 | 112 | func (s *Source) AddApiKeys(keys []string) { 113 | s.apiKeys = keys 114 | } 115 | 116 | func (s *Source) Statistics() subscraping.Statistics { 117 | return subscraping.Statistics{ 118 | Errors: s.errors, 119 | Results: s.results, 120 | TimeTaken: s.timeTaken, 121 | Skipped: s.skipped, 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/sitedossier/sitedossier.go: -------------------------------------------------------------------------------- 1 | // Package sitedossier logic 2 | package sitedossier 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "regexp" 10 | "time" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | // SleepRandIntn is the integer value to get the pseudo-random number 16 | // to sleep before find the next match 17 | const SleepRandIntn = 5 18 | 19 | var reNext = regexp.MustCompile(``) 20 | 21 | // Source is the passive scraping agent 22 | type Source struct { 23 | timeTaken time.Duration 24 | errors int 25 | results int 26 | } 27 | 28 | // Run function returns all subdomains found with the service 29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 30 | results := make(chan subscraping.Result) 31 | s.errors = 0 32 | s.results = 0 33 | 34 | go func() { 35 | defer func(startTime time.Time) { 36 | s.timeTaken = time.Since(startTime) 37 | close(results) 38 | }(time.Now()) 39 | 40 | s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain), results) 41 | }() 42 | 43 | return results 44 | } 45 | 46 | func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, results chan subscraping.Result) { 47 | select { 48 | case <-ctx.Done(): 49 | return 50 | default: 51 | } 52 | 53 | resp, err := session.SimpleGet(ctx, baseURL) 54 | isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound 55 | if err != nil && !isnotfound { 56 | results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} 57 | s.errors++ 58 | session.DiscardHTTPResponse(resp) 59 | return 60 | } 61 | 62 | body, err := io.ReadAll(resp.Body) 63 | if err != nil { 64 | results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} 65 | s.errors++ 66 | resp.Body.Close() 67 | return 68 | } 69 | resp.Body.Close() 70 | 71 | src := string(body) 72 | for _, subdomain := range session.Extractor.Extract(src) { 73 | results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} 74 | s.results++ 75 | } 76 | 77 | match := reNext.FindStringSubmatch(src) 78 | if len(match) > 0 { 79 | s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com%s", match[1]), results) 80 | } 81 | } 82 | 83 | // Name returns the name of the source 84 | func (s *Source) Name() string { 85 | return "sitedossier" 86 | } 87 | 88 | func (s *Source) IsDefault() bool { 89 | return false 90 | } 91 | 92 | func (s *Source) HasRecursiveSupport() bool { 93 | return false 94 | } 95 | 96 | func (s *Source) NeedsKey() bool { 97 | return false 98 | } 99 | 100 | func (s *Source) AddApiKeys(_ []string) { 101 | // no key needed 102 | } 103 | 104 | func (s *Source) Statistics() subscraping.Statistics { 105 | return subscraping.Statistics{ 106 | Errors: s.errors, 107 | Results: s.results, 108 | TimeTaken: s.timeTaken, 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/threatbook/threatbook.go: -------------------------------------------------------------------------------- 1 | // Package threatbook logic 2 | package threatbook 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strconv" 8 | "time" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | type threatBookResponse struct { 16 | ResponseCode int64 `json:"response_code"` 17 | VerboseMsg string `json:"verbose_msg"` 18 | Data struct { 19 | Domain string `json:"domain"` 20 | SubDomains struct { 21 | Total string `json:"total"` 22 | Data []string `json:"data"` 23 | } `json:"sub_domains"` 24 | } `json:"data"` 25 | } 26 | 27 | // Source is the passive scraping agent 28 | type Source struct { 29 | apiKeys []string 30 | timeTaken time.Duration 31 | errors int 32 | results int 33 | skipped bool 34 | } 35 | 36 | // Run function returns all subdomains found with the service 37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 38 | results := make(chan subscraping.Result) 39 | s.errors = 0 40 | s.results = 0 41 | 42 | go func() { 43 | defer func(startTime time.Time) { 44 | s.timeTaken = time.Since(startTime) 45 | close(results) 46 | }(time.Now()) 47 | 48 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 49 | if randomApiKey == "" { 50 | s.skipped = true 51 | return 52 | } 53 | 54 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) 55 | if err != nil && resp == nil { 56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 57 | s.errors++ 58 | session.DiscardHTTPResponse(resp) 59 | return 60 | } 61 | 62 | var response threatBookResponse 63 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 64 | if err != nil { 65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 66 | s.errors++ 67 | resp.Body.Close() 68 | return 69 | } 70 | resp.Body.Close() 71 | 72 | if response.ResponseCode != 0 { 73 | results <- subscraping.Result{ 74 | Source: s.Name(), Type: subscraping.Error, 75 | Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg), 76 | } 77 | s.errors++ 78 | return 79 | } 80 | 81 | total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64) 82 | if err != nil { 83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 84 | s.errors++ 85 | return 86 | } 87 | 88 | if total > 0 { 89 | for _, subdomain := range response.Data.SubDomains.Data { 90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 91 | s.results++ 92 | } 93 | } 94 | }() 95 | 96 | return results 97 | } 98 | 99 | // Name returns the name of the source 100 | func (s *Source) Name() string { 101 | return "threatbook" 102 | } 103 | 104 | func (s *Source) IsDefault() bool { 105 | return false 106 | } 107 | 108 | func (s *Source) HasRecursiveSupport() bool { 109 | return false 110 | } 111 | 112 | func (s *Source) NeedsKey() bool { 113 | return true 114 | } 115 | 116 | func (s *Source) AddApiKeys(keys []string) { 117 | s.apiKeys = keys 118 | } 119 | 120 | func (s *Source) Statistics() subscraping.Statistics { 121 | return subscraping.Statistics{ 122 | Errors: s.errors, 123 | Results: s.results, 124 | TimeTaken: s.timeTaken, 125 | Skipped: s.skipped, 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go: -------------------------------------------------------------------------------- 1 | package threatcrowd 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | // threatCrowdResponse represents the JSON response from the ThreatCrowd API. 15 | type threatCrowdResponse struct { 16 | ResponseCode string `json:"response_code"` 17 | Subdomains []string `json:"subdomains"` 18 | Undercount string `json:"undercount"` 19 | } 20 | 21 | // Source implements the subscraping.Source interface for ThreatCrowd. 22 | type Source struct { 23 | timeTaken time.Duration 24 | errors int 25 | results int 26 | } 27 | 28 | // Run queries the ThreatCrowd API for the given domain and returns found subdomains. 29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 30 | results := make(chan subscraping.Result) 31 | s.errors = 0 32 | s.results = 0 33 | 34 | go func(startTime time.Time) { 35 | defer func() { 36 | s.timeTaken = time.Since(startTime) 37 | close(results) 38 | }() 39 | 40 | url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain) 41 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 42 | if err != nil { 43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 44 | s.errors++ 45 | return 46 | } 47 | 48 | resp, err := session.Client.Do(req) 49 | if err != nil { 50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 51 | s.errors++ 52 | return 53 | } 54 | defer resp.Body.Close() 55 | 56 | if resp.StatusCode != http.StatusOK { 57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} 58 | s.errors++ 59 | return 60 | } 61 | 62 | body, err := io.ReadAll(resp.Body) 63 | if err != nil { 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 65 | s.errors++ 66 | return 67 | } 68 | 69 | var tcResponse threatCrowdResponse 70 | if err := json.Unmarshal(body, &tcResponse); err != nil { 71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 72 | s.errors++ 73 | return 74 | } 75 | 76 | for _, subdomain := range tcResponse.Subdomains { 77 | if subdomain != "" { 78 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 79 | s.results++ 80 | } 81 | } 82 | }(time.Now()) 83 | 84 | return results 85 | } 86 | 87 | // Name returns the name of the source. 88 | func (s *Source) Name() string { 89 | return "threatcrowd" 90 | } 91 | 92 | // IsDefault indicates whether this source is enabled by default. 93 | func (s *Source) IsDefault() bool { 94 | return false 95 | } 96 | 97 | // HasRecursiveSupport indicates if the source supports recursive searches. 98 | func (s *Source) HasRecursiveSupport() bool { 99 | return false 100 | } 101 | 102 | // NeedsKey indicates if the source requires an API key. 103 | func (s *Source) NeedsKey() bool { 104 | return false 105 | } 106 | 107 | // AddApiKeys is a no-op since ThreatCrowd does not require an API key. 108 | func (s *Source) AddApiKeys(_ []string) {} 109 | 110 | // Statistics returns usage statistics. 111 | func (s *Source) Statistics() subscraping.Statistics { 112 | return subscraping.Statistics{ 113 | Errors: s.errors, 114 | Results: s.results, 115 | TimeTaken: s.timeTaken, 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/threatminer/threatminer.go: -------------------------------------------------------------------------------- 1 | // Package threatminer logic 2 | package threatminer 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | type response struct { 15 | StatusCode string `json:"status_code"` 16 | StatusMessage string `json:"status_message"` 17 | Results []string `json:"results"` 18 | } 19 | 20 | // Source is the passive scraping agent 21 | type Source struct { 22 | timeTaken time.Duration 23 | errors int 24 | results int 25 | } 26 | 27 | // Run function returns all subdomains found with the service 28 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 29 | results := make(chan subscraping.Result) 30 | s.errors = 0 31 | s.results = 0 32 | 33 | go func() { 34 | defer func(startTime time.Time) { 35 | s.timeTaken = time.Since(startTime) 36 | close(results) 37 | }(time.Now()) 38 | 39 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | s.errors++ 43 | session.DiscardHTTPResponse(resp) 44 | return 45 | } 46 | 47 | defer resp.Body.Close() 48 | 49 | var data response 50 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 51 | if err != nil { 52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 53 | s.errors++ 54 | return 55 | } 56 | 57 | for _, subdomain := range data.Results { 58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 59 | s.results++ 60 | } 61 | }() 62 | 63 | return results 64 | } 65 | 66 | // Name returns the name of the source 67 | func (s *Source) Name() string { 68 | return "threatminer" 69 | } 70 | 71 | func (s *Source) IsDefault() bool { 72 | return true 73 | } 74 | 75 | func (s *Source) HasRecursiveSupport() bool { 76 | return false 77 | } 78 | 79 | func (s *Source) NeedsKey() bool { 80 | return false 81 | } 82 | 83 | func (s *Source) AddApiKeys(_ []string) { 84 | // no key needed 85 | } 86 | 87 | func (s *Source) Statistics() subscraping.Statistics { 88 | return subscraping.Statistics{ 89 | Errors: s.errors, 90 | Results: s.results, 91 | TimeTaken: s.timeTaken, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/virustotal/virustotal.go: -------------------------------------------------------------------------------- 1 | // Package virustotal logic 2 | package virustotal 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | type response struct { 15 | Data []Object `json:"data"` 16 | Meta Meta `json:"meta"` 17 | } 18 | 19 | type Object struct { 20 | Id string `json:"id"` 21 | } 22 | 23 | type Meta struct { 24 | Cursor string `json:"cursor"` 25 | } 26 | 27 | // Source is the passive scraping agent 28 | type Source struct { 29 | apiKeys []string 30 | timeTaken time.Duration 31 | errors int 32 | results int 33 | skipped bool 34 | } 35 | 36 | // Run function returns all subdomains found with the service 37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 38 | results := make(chan subscraping.Result) 39 | s.errors = 0 40 | s.results = 0 41 | 42 | go func() { 43 | defer func(startTime time.Time) { 44 | s.timeTaken = time.Since(startTime) 45 | close(results) 46 | }(time.Now()) 47 | 48 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 49 | if randomApiKey == "" { 50 | return 51 | } 52 | var cursor string = "" 53 | for { 54 | var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) 55 | if cursor != "" { 56 | url = fmt.Sprintf("%s&cursor=%s", url, cursor) 57 | } 58 | resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey}) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | s.errors++ 62 | session.DiscardHTTPResponse(resp) 63 | return 64 | } 65 | defer resp.Body.Close() 66 | 67 | var data response 68 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 69 | if err != nil { 70 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 71 | s.errors++ 72 | return 73 | } 74 | 75 | for _, subdomain := range data.Data { 76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id} 77 | s.results++ 78 | } 79 | cursor = data.Meta.Cursor 80 | if cursor == "" { 81 | break 82 | } 83 | } 84 | }() 85 | 86 | return results 87 | } 88 | 89 | // Name returns the name of the source 90 | func (s *Source) Name() string { 91 | return "virustotal" 92 | } 93 | 94 | func (s *Source) IsDefault() bool { 95 | return true 96 | } 97 | 98 | func (s *Source) HasRecursiveSupport() bool { 99 | return true 100 | } 101 | 102 | func (s *Source) NeedsKey() bool { 103 | return true 104 | } 105 | 106 | func (s *Source) AddApiKeys(keys []string) { 107 | s.apiKeys = keys 108 | } 109 | 110 | func (s *Source) Statistics() subscraping.Statistics { 111 | return subscraping.Statistics{ 112 | Errors: s.errors, 113 | Results: s.results, 114 | TimeTaken: s.timeTaken, 115 | Skipped: s.skipped, 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go: -------------------------------------------------------------------------------- 1 | // Package waybackarchive logic 2 | package waybackarchive 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "net/url" 9 | "strings" 10 | "time" 11 | 12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 13 | ) 14 | 15 | // Source is the passive scraping agent 16 | type Source struct { 17 | timeTaken time.Duration 18 | errors int 19 | results int 20 | } 21 | 22 | // Run function returns all subdomains found with the service 23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 24 | results := make(chan subscraping.Result) 25 | s.errors = 0 26 | s.results = 0 27 | 28 | go func() { 29 | defer func(startTime time.Time) { 30 | s.timeTaken = time.Since(startTime) 31 | close(results) 32 | }(time.Now()) 33 | 34 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) 35 | if err != nil { 36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 37 | s.errors++ 38 | session.DiscardHTTPResponse(resp) 39 | return 40 | } 41 | 42 | defer resp.Body.Close() 43 | 44 | scanner := bufio.NewScanner(resp.Body) 45 | for scanner.Scan() { 46 | line := scanner.Text() 47 | if line == "" { 48 | continue 49 | } 50 | line, _ = url.QueryUnescape(line) 51 | for _, subdomain := range session.Extractor.Extract(line) { 52 | // fix for triple encoded URL 53 | subdomain = strings.ToLower(subdomain) 54 | subdomain = strings.TrimPrefix(subdomain, "25") 55 | subdomain = strings.TrimPrefix(subdomain, "2f") 56 | 57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 58 | s.results++ 59 | } 60 | } 61 | }() 62 | 63 | return results 64 | } 65 | 66 | // Name returns the name of the source 67 | func (s *Source) Name() string { 68 | return "waybackarchive" 69 | } 70 | 71 | func (s *Source) IsDefault() bool { 72 | return false 73 | } 74 | 75 | func (s *Source) HasRecursiveSupport() bool { 76 | return false 77 | } 78 | 79 | func (s *Source) NeedsKey() bool { 80 | return false 81 | } 82 | 83 | func (s *Source) AddApiKeys(_ []string) { 84 | // no key needed 85 | } 86 | 87 | func (s *Source) Statistics() subscraping.Statistics { 88 | return subscraping.Statistics{ 89 | Errors: s.errors, 90 | Results: s.results, 91 | TimeTaken: s.timeTaken, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go: -------------------------------------------------------------------------------- 1 | // Package whoisxmlapi logic 2 | package whoisxmlapi 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | type response struct { 15 | Search string `json:"search"` 16 | Result Result `json:"result"` 17 | } 18 | 19 | type Result struct { 20 | Count int `json:"count"` 21 | Records []Record `json:"records"` 22 | } 23 | 24 | type Record struct { 25 | Domain string `json:"domain"` 26 | FirstSeen int `json:"firstSeen"` 27 | LastSeen int `json:"lastSeen"` 28 | } 29 | 30 | // Source is the passive scraping agent 31 | type Source struct { 32 | apiKeys []string 33 | timeTaken time.Duration 34 | errors int 35 | results int 36 | skipped bool 37 | } 38 | 39 | // Run function returns all subdomains found with the service 40 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 41 | results := make(chan subscraping.Result) 42 | s.errors = 0 43 | s.results = 0 44 | 45 | go func() { 46 | defer func(startTime time.Time) { 47 | s.timeTaken = time.Since(startTime) 48 | close(results) 49 | }(time.Now()) 50 | 51 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 52 | if randomApiKey == "" { 53 | s.skipped = true 54 | return 55 | } 56 | 57 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) 58 | if err != nil { 59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 60 | s.errors++ 61 | session.DiscardHTTPResponse(resp) 62 | return 63 | } 64 | 65 | var data response 66 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 67 | if err != nil { 68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 69 | s.errors++ 70 | resp.Body.Close() 71 | return 72 | } 73 | 74 | resp.Body.Close() 75 | 76 | for _, record := range data.Result.Records { 77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} 78 | s.results++ 79 | } 80 | }() 81 | 82 | return results 83 | } 84 | 85 | // Name returns the name of the source 86 | func (s *Source) Name() string { 87 | return "whoisxmlapi" 88 | } 89 | 90 | func (s *Source) IsDefault() bool { 91 | return true 92 | } 93 | 94 | func (s *Source) HasRecursiveSupport() bool { 95 | return false 96 | } 97 | 98 | func (s *Source) NeedsKey() bool { 99 | return true 100 | } 101 | 102 | func (s *Source) AddApiKeys(keys []string) { 103 | s.apiKeys = keys 104 | } 105 | 106 | func (s *Source) Statistics() subscraping.Statistics { 107 | return subscraping.Statistics{ 108 | Errors: s.errors, 109 | Results: s.results, 110 | TimeTaken: s.timeTaken, 111 | Skipped: s.skipped, 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go: -------------------------------------------------------------------------------- 1 | package zoomeyeapi 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | "time" 10 | 11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" 12 | ) 13 | 14 | // search results 15 | type zoomeyeResults struct { 16 | Status int `json:"status"` 17 | Total int `json:"total"` 18 | List []struct { 19 | Name string `json:"name"` 20 | Ip []string `json:"ip"` 21 | } `json:"list"` 22 | } 23 | 24 | // Source is the passive scraping agent 25 | type Source struct { 26 | apiKeys []string 27 | timeTaken time.Duration 28 | errors int 29 | results int 30 | skipped bool 31 | } 32 | 33 | // Run function returns all subdomains found with the service 34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 35 | results := make(chan subscraping.Result) 36 | s.errors = 0 37 | s.results = 0 38 | 39 | go func() { 40 | defer func(startTime time.Time) { 41 | s.timeTaken = time.Since(startTime) 42 | close(results) 43 | }(time.Now()) 44 | 45 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) 46 | if randomApiKey == "" { 47 | s.skipped = true 48 | return 49 | } 50 | 51 | randomApiInfo := strings.Split(randomApiKey, ":") 52 | if len(randomApiInfo) != 2 { 53 | s.skipped = true 54 | return 55 | } 56 | host := randomApiInfo[0] 57 | apiKey := randomApiInfo[1] 58 | 59 | headers := map[string]string{ 60 | "API-KEY": apiKey, 61 | "Accept": "application/json", 62 | "Content-Type": "application/json", 63 | } 64 | var pages = 1 65 | for currentPage := 1; currentPage <= pages; currentPage++ { 66 | api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) 67 | resp, err := session.Get(ctx, api, "", headers) 68 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden 69 | if err != nil { 70 | if !isForbidden { 71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 72 | s.errors++ 73 | session.DiscardHTTPResponse(resp) 74 | } 75 | return 76 | } 77 | 78 | var res zoomeyeResults 79 | err = json.NewDecoder(resp.Body).Decode(&res) 80 | 81 | if err != nil { 82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 83 | s.errors++ 84 | _ = resp.Body.Close() 85 | return 86 | } 87 | _ = resp.Body.Close() 88 | pages = int(res.Total/1000) + 1 89 | for _, r := range res.List { 90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} 91 | s.results++ 92 | } 93 | } 94 | }() 95 | 96 | return results 97 | } 98 | 99 | // Name returns the name of the source 100 | func (s *Source) Name() string { 101 | return "zoomeyeapi" 102 | } 103 | 104 | func (s *Source) IsDefault() bool { 105 | return false 106 | } 107 | 108 | func (s *Source) HasRecursiveSupport() bool { 109 | return false 110 | } 111 | 112 | func (s *Source) NeedsKey() bool { 113 | return true 114 | } 115 | 116 | func (s *Source) AddApiKeys(keys []string) { 117 | s.apiKeys = keys 118 | } 119 | 120 | func (s *Source) Statistics() subscraping.Statistics { 121 | return subscraping.Statistics{ 122 | Errors: s.errors, 123 | Results: s.results, 124 | TimeTaken: s.timeTaken, 125 | Skipped: s.skipped, 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/types.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/projectdiscovery/ratelimit" 9 | mapsutil "github.com/projectdiscovery/utils/maps" 10 | ) 11 | 12 | type CtxArg string 13 | 14 | const ( 15 | CtxSourceArg CtxArg = "source" 16 | ) 17 | 18 | type CustomRateLimit struct { 19 | Custom mapsutil.SyncLockMap[string, uint] 20 | } 21 | 22 | // BasicAuth request's Authorization header 23 | type BasicAuth struct { 24 | Username string 25 | Password string 26 | } 27 | 28 | // Statistics contains statistics about the scraping process 29 | type Statistics struct { 30 | TimeTaken time.Duration 31 | Errors int 32 | Results int 33 | Skipped bool 34 | } 35 | 36 | // Source is an interface inherited by each passive source 37 | type Source interface { 38 | // Run takes a domain as argument and a session object 39 | // which contains the extractor for subdomains, http client 40 | // and other stuff. 41 | Run(context.Context, string, *Session) <-chan Result 42 | 43 | // Name returns the name of the source. It is preferred to use lower case names. 44 | Name() string 45 | 46 | // IsDefault returns true if the current source should be 47 | // used as part of the default execution. 48 | IsDefault() bool 49 | 50 | // HasRecursiveSupport returns true if the current source 51 | // accepts subdomains (e.g. subdomain.domain.tld), 52 | // not just root domains. 53 | HasRecursiveSupport() bool 54 | 55 | // NeedsKey returns true if the source requires an API key 56 | NeedsKey() bool 57 | 58 | AddApiKeys([]string) 59 | 60 | // Statistics returns the scrapping statistics for the source 61 | Statistics() Statistics 62 | } 63 | 64 | // SubdomainExtractor is an interface that defines the contract for subdomain extraction. 65 | type SubdomainExtractor interface { 66 | Extract(text string) []string 67 | } 68 | 69 | // Session is the option passed to the source, an option is created 70 | // uniquely for each source. 71 | type Session struct { 72 | //SubdomainExtractor 73 | Extractor SubdomainExtractor 74 | // Client is the current http client 75 | Client *http.Client 76 | // Rate limit instance 77 | MultiRateLimiter *ratelimit.MultiLimiter 78 | } 79 | 80 | // Result is a result structure returned by a source 81 | type Result struct { 82 | Type ResultType 83 | Source string 84 | Value string 85 | Error error 86 | } 87 | 88 | // ResultType is the type of result returned by the source 89 | type ResultType int 90 | 91 | // Types of results returned by the source 92 | const ( 93 | Subdomain ResultType = iota 94 | Error 95 | ) 96 | -------------------------------------------------------------------------------- /v2/pkg/subscraping/utils.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "math/rand" 5 | "strings" 6 | 7 | "github.com/projectdiscovery/gologger" 8 | ) 9 | 10 | const MultipleKeyPartsLength = 2 11 | 12 | func PickRandom[T any](v []T, sourceName string) T { 13 | var result T 14 | length := len(v) 15 | if length == 0 { 16 | gologger.Debug().Msgf("Cannot use the %s source because there was no API key/secret defined for it.", sourceName) 17 | return result 18 | } 19 | return v[rand.Intn(length)] 20 | } 21 | 22 | func CreateApiKeys[T any](keys []string, provider func(k, v string) T) []T { 23 | var result []T 24 | for _, key := range keys { 25 | if keyPartA, keyPartB, ok := createMultiPartKey(key); ok { 26 | result = append(result, provider(keyPartA, keyPartB)) 27 | } 28 | } 29 | return result 30 | } 31 | 32 | func createMultiPartKey(key string) (keyPartA, keyPartB string, ok bool) { 33 | parts := strings.Split(key, ":") 34 | ok = len(parts) == MultipleKeyPartsLength 35 | 36 | if ok { 37 | keyPartA = parts[0] 38 | keyPartB = parts[1] 39 | } 40 | 41 | return 42 | } 43 | -------------------------------------------------------------------------------- /v2/pkg/testutils/integration.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "strings" 8 | ) 9 | 10 | func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]string, error) { 11 | cmd := exec.Command("bash", "-c") 12 | cmdLine := fmt.Sprintf("echo %s | %s", domain, "./subfinder ") 13 | cmdLine += strings.Join(extra, " ") 14 | cmd.Args = append(cmd.Args, cmdLine) 15 | if debug { 16 | cmd.Args = append(cmd.Args, "-v") 17 | cmd.Stderr = os.Stderr 18 | fmt.Println(cmd.String()) 19 | } else { 20 | cmd.Args = append(cmd.Args, "-silent") 21 | } 22 | data, err := cmd.Output() 23 | if debug { 24 | fmt.Println(string(data)) 25 | } 26 | if err != nil { 27 | return nil, err 28 | } 29 | var parts []string 30 | items := strings.Split(string(data), "\n") 31 | for _, i := range items { 32 | if i != "" { 33 | parts = append(parts, i) 34 | } 35 | } 36 | return parts, nil 37 | } 38 | 39 | // TestCase is a single integration test case 40 | type TestCase interface { 41 | Execute() error 42 | } 43 | --------------------------------------------------------------------------------