├── .github
├── ISSUE_TEMPLATE
│ ├── bug_report.md
│ ├── config.yml
│ ├── feature_request.md
│ └── issue-report.md
├── dependabot.yml
├── release.yml
└── workflows
│ ├── build-test.yml
│ ├── codeql-analysis.yml
│ ├── compat-checks.yaml
│ ├── dep-auto-merge.yml
│ ├── dockerhub-push.yml
│ ├── release-binary.yml
│ └── release-test.yml
├── .gitignore
├── DISCLAIMER.md
├── Dockerfile
├── LICENSE.md
├── README.md
├── THANKS.md
├── static
├── subfinder-logo.png
└── subfinder-run.png
└── v2
├── .goreleaser.yml
├── Makefile
├── cmd
└── subfinder
│ └── main.go
├── examples
└── main.go
├── go.mod
├── go.sum
└── pkg
├── passive
├── doc.go
├── passive.go
├── sources.go
├── sources_test.go
├── sources_w_auth_test.go
└── sources_wo_auth_test.go
├── resolve
├── client.go
├── doc.go
└── resolve.go
├── runner
├── banners.go
├── config.go
├── doc.go
├── enumerate.go
├── enumerate_test.go
├── initialize.go
├── options.go
├── outputter.go
├── runner.go
├── stats.go
├── util.go
└── validate.go
├── subscraping
├── agent.go
├── doc.go
├── extractor.go
├── sources
│ ├── alienvault
│ │ └── alienvault.go
│ ├── anubis
│ │ └── anubis.go
│ ├── bevigil
│ │ └── bevigil.go
│ ├── bufferover
│ │ └── bufferover.go
│ ├── builtwith
│ │ └── builtwith.go
│ ├── c99
│ │ └── c99.go
│ ├── censys
│ │ └── censys.go
│ ├── certspotter
│ │ └── certspotter.go
│ ├── chaos
│ │ └── chaos.go
│ ├── chinaz
│ │ └── chinaz.go
│ ├── commoncrawl
│ │ └── commoncrawl.go
│ ├── crtsh
│ │ └── crtsh.go
│ ├── digitalyama
│ │ └── digitalyama.go
│ ├── digitorus
│ │ └── digitorus.go
│ ├── dnsdb
│ │ └── dnsdb.go
│ ├── dnsdumpster
│ │ └── dnsdumpster.go
│ ├── dnsrepo
│ │ └── dnsrepo.go
│ ├── facebook
│ │ ├── ctlogs.go
│ │ ├── ctlogs_test.go
│ │ └── types.go
│ ├── fofa
│ │ └── fofa.go
│ ├── fullhunt
│ │ └── fullhunt.go
│ ├── github
│ │ ├── github.go
│ │ └── tokenmanager.go
│ ├── gitlab
│ │ └── gitlab.go
│ ├── hackertarget
│ │ └── hackertarget.go
│ ├── hudsonrock
│ │ └── hudsonrock.go
│ ├── hunter
│ │ └── hunter.go
│ ├── intelx
│ │ └── intelx.go
│ ├── leakix
│ │ └── leakix.go
│ ├── netlas
│ │ └── netlas.go
│ ├── pugrecon
│ │ └── pugrecon.go
│ ├── quake
│ │ └── quake.go
│ ├── rapiddns
│ │ └── rapiddns.go
│ ├── reconcloud
│ │ └── reconcloud.go
│ ├── redhuntlabs
│ │ └── redhuntlabs.go
│ ├── riddler
│ │ └── riddler.go
│ ├── robtex
│ │ └── robtext.go
│ ├── rsecloud
│ │ └── rsecloud.go
│ ├── securitytrails
│ │ └── securitytrails.go
│ ├── shodan
│ │ └── shodan.go
│ ├── sitedossier
│ │ └── sitedossier.go
│ ├── threatbook
│ │ └── threatbook.go
│ ├── threatcrowd
│ │ └── threatcrowd.go
│ ├── threatminer
│ │ └── threatminer.go
│ ├── virustotal
│ │ └── virustotal.go
│ ├── waybackarchive
│ │ └── waybackarchive.go
│ ├── whoisxmlapi
│ │ └── whoisxmlapi.go
│ └── zoomeyeapi
│ │ └── zoomeyeapi.go
├── types.go
└── utils.go
└── testutils
└── integration.go
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Bug report
3 | about: Create a report to help us improve
4 | title: "[Issue] "
5 | labels: ''
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Describe the bug**
11 | A clear and concise description of what the bug is.
12 |
13 | **Subfinder version**
14 | Include the version of subfinder you are using, `subfinder -version`
15 |
16 | **Complete command you used to reproduce this**
17 |
18 |
19 | **Screenshots**
20 | Add screenshots of the error for a better context.
21 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
1 | blank_issues_enabled: false
2 |
3 | contact_links:
4 | - name: Ask an question / advise on using subfinder
5 | url: https://github.com/projectdiscovery/subfinder/discussions/categories/q-a
6 | about: Ask a question or request support for using subfinder
7 |
8 | - name: Share idea / feature to discuss for subfinder
9 | url: https://github.com/projectdiscovery/subfinder/discussions/categories/ideas
10 | about: Share idea / feature to discuss for subfinder
11 |
12 | - name: Connect with PD Team (Discord)
13 | url: https://discord.gg/projectdiscovery
14 | about: Connect with PD Team for direct communication
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Feature request
3 | about: Request feature to implement in this project
4 | labels: 'Type: Enhancement'
5 | ---
6 |
7 | <!--
8 | 1. Please make sure to provide a detailed description with all the relevant information that might be required to start working on this feature.
9 | 2. In case you are not sure about your request or whether the particular feature is already supported or not, please start a discussion instead.
10 | 3. GitHub Discussion: https://github.com/projectdiscovery/subfinder/discussions/categories/ideas
11 | 4. Join our discord server at https://discord.gg/projectdiscovery to discuss the idea on the #subfinder channel.
12 | -->
13 |
14 | ### Please describe your feature request:
15 | <!-- A clear and concise description of feature to implement -->
16 |
17 | ### Describe the use case of this feature:
18 | <!-- A clear and concise description of the feature request's motivation and the use-cases in which it could be useful. -->
19 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/issue-report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Issue report
3 | about: Create a report to help us to improve the project
4 | labels: 'Type: Bug'
5 |
6 | ---
7 |
8 | <!--
9 | 1. Please search to see if an issue already exists for the bug you encountered.
10 | 2. For support requests, FAQs or "How to" questions, please use the GitHub Discussions section instead - https://github.com/projectdiscovery/subfinder/discussions or
11 | 3. Join our discord server at https://discord.gg/projectdiscovery and post the question on the #subfinder channel.
12 | -->
13 |
14 | <!-- ISSUES MISSING IMPORTANT INFORMATION MAY BE CLOSED WITHOUT INVESTIGATION. -->
15 |
16 | ### Subfinder version:
17 | <!-- You can find current version of subfinder with "subfinder -version" -->
18 | <!-- We only accept issues that are reproducible on the latest version of subfinder. -->
19 | <!-- You can find the latest version of project at https://github.com/projectdiscovery/subfinder/releases/ -->
20 |
21 | ### Current Behavior:
22 | <!-- A concise description of what you're experiencing. -->
23 |
24 | ### Expected Behavior:
25 | <!-- A concise description of what you expected to happen. -->
26 |
27 | ### Steps To Reproduce:
28 | <!--
29 | Example: steps to reproduce the behavior:
30 | 1. Run 'subfinder ..'
31 | 2. See error...
32 | -->
33 |
34 |
35 | ### Anything else:
36 | <!-- Links? References? Screnshots? Anything that will give us more context about the issue that you are encountering! -->
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # To get started with Dependabot version updates, you'll need to specify which
2 | # package ecosystems to update and where the package manifests are located.
3 | # Please see the documentation for all configuration options:
4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
5 |
6 | version: 2
7 | updates:
8 |
9 | # Maintain dependencies for go modules
10 | - package-ecosystem: "gomod"
11 | directory: "v2/"
12 | schedule:
13 | interval: "weekly"
14 | target-branch: "dev"
15 | commit-message:
16 | prefix: "chore"
17 | include: "scope"
18 | allow:
19 | - dependency-name: "github.com/projectdiscovery/*"
20 | groups:
21 | modules:
22 | patterns: ["github.com/projectdiscovery/*"]
23 | labels:
24 | - "Type: Maintenance"
25 |
26 | # # Maintain dependencies for GitHub Actions
27 | # - package-ecosystem: "github-actions"
28 | # directory: "/"
29 | # schedule:
30 | # interval: "weekly"
31 | # target-branch: "dev"
32 | # commit-message:
33 | # prefix: "chore"
34 | # include: "scope"
35 | # labels:
36 | # - "Type: Maintenance"
37 | #
38 | # # Maintain dependencies for docker
39 | # - package-ecosystem: "docker"
40 | # directory: "/"
41 | # schedule:
42 | # interval: "weekly"
43 | # target-branch: "dev"
44 | # commit-message:
45 | # prefix: "chore"
46 | # include: "scope"
47 | # labels:
48 | # - "Type: Maintenance"
--------------------------------------------------------------------------------
/.github/release.yml:
--------------------------------------------------------------------------------
1 | changelog:
2 | exclude:
3 | authors:
4 | - dependabot
5 | categories:
6 | - title: 🎉 New Features
7 | labels:
8 | - "Type: Enhancement"
9 | - title: 🐞 Bug Fixes
10 | labels:
11 | - "Type: Bug"
12 | - title: 🔨 Maintenance
13 | labels:
14 | - "Type: Maintenance"
15 | - title: Other Changes
16 | labels:
17 | - "*"
--------------------------------------------------------------------------------
/.github/workflows/build-test.yml:
--------------------------------------------------------------------------------
1 | name: 🔨 Build Test
2 |
3 | on:
4 | pull_request:
5 | paths:
6 | - "**.go"
7 | - "**.mod"
8 | workflow_dispatch:
9 | inputs:
10 | short:
11 | description: "Use -short flag for tests"
12 | required: false
13 | type: boolean
14 | default: false
15 |
16 | jobs:
17 | lint:
18 | name: Lint Test
19 | if: "${{ !endsWith(github.actor, '[bot]') }}"
20 | runs-on: ubuntu-latest
21 | steps:
22 | - uses: actions/checkout@v4
23 | - uses: projectdiscovery/actions/setup/go@v1
24 | with:
25 | go-version-file: v2/go.mod
26 | - name: Run golangci-lint
27 | uses: golangci/golangci-lint-action@v8
28 | with:
29 | version: latest
30 | args: --timeout 5m
31 | working-directory: v2/
32 |
33 | build:
34 | name: Test Builds
35 | needs: [lint]
36 | runs-on: ${{ matrix.os }}
37 | strategy:
38 | matrix:
39 | os: [ubuntu-latest, windows-latest, macOS-13]
40 | steps:
41 | - uses: actions/checkout@v4
42 | - uses: projectdiscovery/actions/setup/go@v1
43 | with:
44 | go-version-file: v2/go.mod
45 | - run: go build ./...
46 | working-directory: v2/
47 |
48 | - name: Run tests
49 | env:
50 | BEVIGIL_API_KEY: ${{secrets.BEVIGIL_API_KEY}}
51 | BINARYEDGE_API_KEY: ${{secrets.BINARYEDGE_API_KEY}}
52 | BUFFEROVER_API_KEY: ${{secrets.BUFFEROVER_API_KEY}}
53 | C99_API_KEY: ${{secrets.C99_API_KEY}}
54 | CENSYS_API_KEY: ${{secrets.CENSYS_API_KEY}}
55 | CERTSPOTTER_API_KEY: ${{secrets.CERTSPOTTER_API_KEY}}
56 | CHAOS_API_KEY: ${{secrets.CHAOS_API_KEY}}
57 | CHINAZ_API_KEY: ${{secrets.CHINAZ_API_KEY}}
58 | DNSDB_API_KEY: ${{secrets.DNSDB_API_KEY}}
59 | DNSREPO_API_KEY: ${{secrets.DNSREPO_API_KEY}}
60 | FOFA_API_KEY: ${{secrets.FOFA_API_KEY}}
61 | FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}}
62 | GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}}
63 | HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}}
64 | INTELX_API_KEY: ${{secrets.INTELX_API_KEY}}
65 | LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}}
66 | QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}}
67 | ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}}
68 | SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}}
69 | SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
70 | THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}}
71 | VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}}
72 | WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}}
73 | ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}}
74 | uses: nick-invision/retry@v2
75 | with:
76 | timeout_seconds: 360
77 | max_attempts: 3
78 | command: cd v2; go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }}
79 |
80 | - name: Race Condition Tests
81 | run: go build -race ./...
82 | working-directory: v2/
83 |
84 | - name: Run Example
85 | run: go run .
86 | working-directory: v2/examples
87 |
--------------------------------------------------------------------------------
/.github/workflows/codeql-analysis.yml:
--------------------------------------------------------------------------------
1 | name: 🚨 CodeQL Analysis
2 |
3 | on:
4 | workflow_dispatch:
5 | pull_request:
6 | paths:
7 | - '**.go'
8 | - '**.mod'
9 |
10 | jobs:
11 | analyze:
12 | name: Analyze
13 | runs-on: ubuntu-latest
14 | permissions:
15 | actions: read
16 | contents: read
17 | security-events: write
18 |
19 | strategy:
20 | fail-fast: false
21 | matrix:
22 | language: [ 'go' ]
23 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
24 |
25 | steps:
26 | - name: Checkout repository
27 | uses: actions/checkout@v3
28 |
29 | # Initializes the CodeQL tools for scanning.
30 | - name: Initialize CodeQL
31 | uses: github/codeql-action/init@v2
32 | with:
33 | languages: ${{ matrix.language }}
34 |
35 | - name: Autobuild
36 | uses: github/codeql-action/autobuild@v2
37 |
38 | - name: Perform CodeQL Analysis
39 | uses: github/codeql-action/analyze@v2
--------------------------------------------------------------------------------
/.github/workflows/compat-checks.yaml:
--------------------------------------------------------------------------------
1 | name: ♾️ Compatibility Checks
2 |
3 | on:
4 | pull_request:
5 | types: [opened, synchronize]
6 | branches:
7 | - dev
8 |
9 | jobs:
10 | check:
11 | if: github.actor == 'dependabot[bot]'
12 | runs-on: ubuntu-latest
13 | permissions:
14 | contents: write
15 | steps:
16 | - uses: actions/checkout@v4
17 | - uses: projectdiscovery/actions/setup/go/compat-checks@master
18 | with:
19 | go-version-file: 'v2/go.mod'
20 |
21 |
--------------------------------------------------------------------------------
/.github/workflows/dep-auto-merge.yml:
--------------------------------------------------------------------------------
1 | name: 🤖 dep auto merge
2 |
3 | on:
4 | pull_request:
5 | branches:
6 | - dev
7 | workflow_dispatch:
8 |
9 | permissions:
10 | pull-requests: write
11 | issues: write
12 | repository-projects: write
13 |
14 | jobs:
15 | automerge:
16 | runs-on: ubuntu-latest
17 | if: github.actor == 'dependabot[bot]'
18 | steps:
19 | - uses: actions/checkout@v3
20 | with:
21 | token: ${{ secrets.DEPENDABOT_PAT }}
22 |
23 | - uses: ahmadnassri/action-dependabot-auto-merge@v2
24 | with:
25 | github-token: ${{ secrets.DEPENDABOT_PAT }}
26 | target: all
--------------------------------------------------------------------------------
/.github/workflows/dockerhub-push.yml:
--------------------------------------------------------------------------------
1 | name: 🌥 Docker Push
2 |
3 | on:
4 | workflow_run:
5 | workflows: ["🎉 Release Binary"]
6 | types:
7 | - completed
8 | workflow_dispatch:
9 |
10 | jobs:
11 | docker:
12 | runs-on: ubuntu-latest-16-cores
13 | steps:
14 | - name: Checkout
15 | uses: actions/checkout@v3
16 |
17 | - name: Get Github tag
18 | id: meta
19 | run: |
20 | curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT
21 |
22 | - name: Set up QEMU
23 | uses: docker/setup-qemu-action@v2
24 |
25 | - name: Set up Docker Buildx
26 | uses: docker/setup-buildx-action@v2
27 |
28 | - name: Login to DockerHub
29 | uses: docker/login-action@v2
30 | with:
31 | username: ${{ secrets.DOCKER_USERNAME }}
32 | password: ${{ secrets.DOCKER_TOKEN }}
33 |
34 | - name: Build and push
35 | uses: docker/build-push-action@v4
36 | with:
37 | context: .
38 | platforms: linux/amd64,linux/arm64,linux/arm
39 | push: true
40 | tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }}
--------------------------------------------------------------------------------
/.github/workflows/release-binary.yml:
--------------------------------------------------------------------------------
1 | name: 🎉 Release Binary
2 |
3 | on:
4 | push:
5 | tags:
6 | - v*
7 | workflow_dispatch:
8 |
9 | jobs:
10 | release:
11 | runs-on: ubuntu-latest-16-cores
12 | steps:
13 | - name: "Check out code"
14 | uses: actions/checkout@v3
15 | with:
16 | fetch-depth: 0
17 |
18 | - name: "Set up Go"
19 | uses: actions/setup-go@v4
20 | with:
21 | go-version: 1.21.x
22 |
23 | - name: "Create release on GitHub"
24 | uses: goreleaser/goreleaser-action@v3
25 | with:
26 | args: "release --clean"
27 | version: latest
28 | workdir: v2/
29 | env:
30 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
31 | SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
32 | DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}"
33 | DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}"
--------------------------------------------------------------------------------
/.github/workflows/release-test.yml:
--------------------------------------------------------------------------------
1 | name: 🔨 Release Test
2 |
3 | on:
4 | pull_request:
5 | paths:
6 | - '**.go'
7 | - '**.mod'
8 | workflow_dispatch:
9 |
10 | jobs:
11 | release-test:
12 | runs-on: ubuntu-latest-16-cores
13 | steps:
14 | - name: "Check out code"
15 | uses: actions/checkout@v3
16 | with:
17 | fetch-depth: 0
18 |
19 | - name: Set up Go
20 | uses: actions/setup-go@v4
21 | with:
22 | go-version: 1.21.x
23 |
24 | - name: release test
25 | uses: goreleaser/goreleaser-action@v4
26 | with:
27 | args: "release --clean --snapshot"
28 | version: latest
29 | workdir: v2
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | cmd/subfinder/subfinder
3 | # subfinder binary when built with `go build`
4 | v2/cmd/subfinder/subfinder
5 | # subfinder binary when built with `make`
6 | v2/subfinder
7 | vendor/
8 | .idea
9 | .devcontainer
10 | .vscode
11 | dist
--------------------------------------------------------------------------------
/DISCLAIMER.md:
--------------------------------------------------------------------------------
1 | ## Disclaimer
2 |
3 | Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services:
4 |
5 | - Bufferover: https://tls.bufferover.run
6 | - CommonCrawl: https://commoncrawl.org/terms-of-use/full
7 | - certspotter: https://sslmate.com/terms
8 | - dnsdumpster: https://hackertarget.com/terms
9 | - Google Transparency: https://policies.google.com/terms
10 | - Alienvault: https://www.alienvault.com/terms/website-terms-of-use07may2018
11 |
12 | ---
13 |
14 | You expressly understand and agree that Subfinder (creators and contributors) shall not be liable for any damages or losses resulting from your use of this tool or third-party products that use it.
15 |
16 | Creators aren't in charge of any and have/has no responsibility for any kind of:
17 |
18 | - Unlawful or illegal use of the tool.
19 | - Legal or Law infringement (acted in any country, state, municipality, place) by third parties and users.
20 | - Act against ethical and / or human moral, ethic, and peoples and cultures of the world.
21 | - Malicious act, capable of causing damage to third parties, promoted or distributed by third parties or the user through this tool.
22 |
23 |
24 | ### Contact
25 |
26 | Please contact at contact@projectdiscovery.io for any questions.
27 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | # Build
2 | FROM golang:1.24-alpine AS build-env
3 | RUN apk add build-base
4 | WORKDIR /app
5 | COPY . /app
6 | WORKDIR /app/v2
7 | RUN go mod download
8 | RUN go build ./cmd/subfinder
9 |
10 | # Release
11 | FROM alpine:latest
12 | RUN apk upgrade --no-cache \
13 | && apk add --no-cache bind-tools ca-certificates
14 | COPY --from=build-env /app/v2/subfinder /usr/local/bin/
15 |
16 | ENTRYPOINT ["subfinder"]
17 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 ProjectDiscovery, Inc.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/THANKS.md:
--------------------------------------------------------------------------------
1 | ### Thanks
2 |
3 | Many people have contributed to subfinder making it a wonderful tool either by making a pull request fixing some stuff or giving generous donations to support the further development of this tool. Here, we recognize these persons and thank them.
4 |
5 | - All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/subfinder/graphs/contributors) who made subfinder what it is.
6 |
7 | We'd like to thank some additional amazing people, who contributed a lot in subfinder's journey -
8 |
9 | - [@vzamanillo](https://github.com/vzamanillo) - For adding multiple features and overall project improvements.
10 | - [@infosec-au](https://github.com/infosec-au) - Donating to the project.
11 | - [@codingo](https://github.com/codingo) - Initial work on the project, managing it, lot of work!
12 | - [@picatz](https://github.com/picatz) - Improving the structure of the project a lot. New ideas!
--------------------------------------------------------------------------------
/static/subfinder-logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/projectdiscovery/subfinder/a3674edd509695a4ee11313a768f003ca0da0da1/static/subfinder-logo.png
--------------------------------------------------------------------------------
/static/subfinder-run.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/projectdiscovery/subfinder/a3674edd509695a4ee11313a768f003ca0da0da1/static/subfinder-run.png
--------------------------------------------------------------------------------
/v2/.goreleaser.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 |
3 | before:
4 | hooks:
5 | - go mod tidy
6 |
7 | builds:
8 | - env:
9 | - CGO_ENABLED=0
10 | goos:
11 | - windows
12 | - linux
13 | - darwin
14 | goarch:
15 | - amd64
16 | - 386
17 | - arm
18 | - arm64
19 |
20 | ignore:
21 | - goos: darwin
22 | goarch: '386'
23 | - goos: windows
24 | goarch: 'arm'
25 |
26 | binary: '{{ .ProjectName }}'
27 | main: cmd/subfinder/main.go
28 |
29 | archives:
30 | - formats:
31 | - zip
32 | name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
33 |
34 | checksum:
35 | algorithm: sha256
36 |
37 | announce:
38 | slack:
39 | enabled: true
40 | channel: '#release'
41 | username: GoReleaser
42 | message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}'
43 |
44 | discord:
45 | enabled: true
46 | message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}'
47 |
--------------------------------------------------------------------------------
/v2/Makefile:
--------------------------------------------------------------------------------
1 | # Go parameters
2 | GOCMD=go
3 | GOBUILD=$(GOCMD) build
4 | GOMOD=$(GOCMD) mod
5 | GOTEST=$(GOCMD) test
6 | GOFLAGS := -v
7 | LDFLAGS := -s -w
8 |
9 | ifneq ($(shell go env GOOS),darwin)
10 | LDFLAGS := -extldflags "-static"
11 | endif
12 |
13 | all: build
14 | build:
15 | $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "subfinder" cmd/subfinder/main.go
16 | test:
17 | $(GOTEST) $(GOFLAGS) ./...
18 | tidy:
19 | $(GOMOD) tidy
20 |
--------------------------------------------------------------------------------
/v2/cmd/subfinder/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/projectdiscovery/subfinder/v2/pkg/runner"
5 | // Attempts to increase the OS file descriptors - Fail silently
6 | _ "github.com/projectdiscovery/fdmax/autofdmax"
7 | "github.com/projectdiscovery/gologger"
8 | )
9 |
10 | func main() {
11 | // Parse the command line flags and read config files
12 | options := runner.ParseOptions()
13 |
14 | newRunner, err := runner.NewRunner(options)
15 | if err != nil {
16 | gologger.Fatal().Msgf("Could not create runner: %s\n", err)
17 | }
18 |
19 | err = newRunner.RunEnumeration()
20 | if err != nil {
21 | gologger.Fatal().Msgf("Could not run enumeration: %s\n", err)
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/v2/examples/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "io"
7 | "log"
8 |
9 | "github.com/projectdiscovery/subfinder/v2/pkg/runner"
10 | )
11 |
12 | func main() {
13 | subfinderOpts := &runner.Options{
14 | Threads: 10, // Thread controls the number of threads to use for active enumerations
15 | Timeout: 30, // Timeout is the seconds to wait for sources to respond
16 | MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
17 | // ResultCallback: func(s *resolve.HostEntry) {
18 | // callback function executed after each unique subdomain is found
19 | // },
20 | // ProviderConfig: "your_provider_config.yaml",
21 | // and other config related options
22 | }
23 |
24 | // disable timestamps in logs / configure logger
25 | log.SetFlags(0)
26 |
27 | subfinder, err := runner.NewRunner(subfinderOpts)
28 | if err != nil {
29 | log.Fatalf("failed to create subfinder runner: %v", err)
30 | }
31 |
32 | output := &bytes.Buffer{}
33 | var sourceMap map[string]map[string]struct{}
34 | // To run subdomain enumeration on a single domain
35 | if sourceMap, err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil {
36 | log.Fatalf("failed to enumerate single domain: %v", err)
37 | }
38 |
39 | // To run subdomain enumeration on a list of domains from file/reader
40 | // file, err := os.Open("domains.txt")
41 | // if err != nil {
42 | // log.Fatalf("failed to open domains file: %v", err)
43 | // }
44 | // defer file.Close()
45 | // if err = subfinder.EnumerateMultipleDomainsWithCtx(context.Background(), file, []io.Writer{output}); err != nil {
46 | // log.Fatalf("failed to enumerate subdomains from file: %v", err)
47 | // }
48 |
49 | // print the output
50 | log.Println(output.String())
51 |
52 | // Or use sourceMap to access the results in your application
53 | for subdomain, sources := range sourceMap {
54 | sourcesList := make([]string, 0, len(sources))
55 | for source := range sources {
56 | sourcesList = append(sourcesList, source)
57 | }
58 | log.Printf("%s %s (%d)\n", subdomain, sourcesList, len(sources))
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/v2/pkg/passive/doc.go:
--------------------------------------------------------------------------------
1 | // Package passive provides capability for doing passive subdomain
2 | // enumeration on targets.
3 | package passive
4 |
--------------------------------------------------------------------------------
/v2/pkg/passive/passive.go:
--------------------------------------------------------------------------------
1 | package passive
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "math"
7 | "sort"
8 | "strings"
9 | "sync"
10 | "time"
11 |
12 | "github.com/projectdiscovery/ratelimit"
13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
14 | )
15 |
16 | type EnumerationOptions struct {
17 | customRateLimiter *subscraping.CustomRateLimit
18 | }
19 |
20 | type EnumerateOption func(opts *EnumerationOptions)
21 |
22 | func WithCustomRateLimit(crl *subscraping.CustomRateLimit) EnumerateOption {
23 | return func(opts *EnumerationOptions) {
24 | opts.customRateLimiter = crl
25 | }
26 | }
27 |
28 | // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context
29 | func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result {
30 | return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime, options...)
31 | }
32 |
33 | // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain
34 | func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result {
35 | results := make(chan subscraping.Result)
36 |
37 | go func() {
38 | defer close(results)
39 |
40 | var enumerateOptions EnumerationOptions
41 | for _, enumerateOption := range options {
42 | enumerateOption(&enumerateOptions)
43 | }
44 |
45 | multiRateLimiter, err := a.buildMultiRateLimiter(ctx, rateLimit, enumerateOptions.customRateLimiter)
46 | if err != nil {
47 | results <- subscraping.Result{
48 | Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err),
49 | }
50 | return
51 | }
52 | session, err := subscraping.NewSession(domain, proxy, multiRateLimiter, timeout)
53 | if err != nil {
54 | results <- subscraping.Result{
55 | Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err),
56 | }
57 | return
58 | }
59 | defer session.Close()
60 |
61 | ctx, cancel := context.WithTimeout(ctx, maxEnumTime)
62 |
63 | wg := &sync.WaitGroup{}
64 | // Run each source in parallel on the target domain
65 | for _, runner := range a.sources {
66 | wg.Add(1)
67 | go func(source subscraping.Source) {
68 | ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name())
69 | for resp := range source.Run(ctxWithValue, domain, session) {
70 | results <- resp
71 | }
72 | wg.Done()
73 | }(runner)
74 | }
75 | wg.Wait()
76 | cancel()
77 | }()
78 | return results
79 | }
80 |
81 | func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) {
82 | var multiRateLimiter *ratelimit.MultiLimiter
83 | var err error
84 | for _, source := range a.sources {
85 | var rl uint
86 | if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok {
87 | rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit)
88 | }
89 |
90 | if rl > 0 {
91 | multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second)
92 | } else {
93 | multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond)
94 | }
95 |
96 | if err != nil {
97 | break
98 | }
99 | }
100 | return multiRateLimiter, err
101 | }
102 |
103 | func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint {
104 | if sourceRateLimit > 0 {
105 | return sourceRateLimit
106 | }
107 | return defaultRateLimit
108 | }
109 |
110 | func addRateLimiter(ctx context.Context, multiRateLimiter *ratelimit.MultiLimiter, key string, maxCount uint, duration time.Duration) (*ratelimit.MultiLimiter, error) {
111 | if multiRateLimiter == nil {
112 | mrl, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{
113 | Key: key,
114 | IsUnlimited: maxCount == math.MaxUint32,
115 | MaxCount: maxCount,
116 | Duration: duration,
117 | })
118 | return mrl, err
119 | }
120 | err := multiRateLimiter.Add(&ratelimit.Options{
121 | Key: key,
122 | IsUnlimited: maxCount == math.MaxUint32,
123 | MaxCount: maxCount,
124 | Duration: duration,
125 | })
126 | return multiRateLimiter, err
127 | }
128 |
129 | func (a *Agent) GetStatistics() map[string]subscraping.Statistics {
130 | stats := make(map[string]subscraping.Statistics)
131 | sort.Slice(a.sources, func(i, j int) bool {
132 | return a.sources[i].Name() > a.sources[j].Name()
133 | })
134 |
135 | for _, source := range a.sources {
136 | stats[source.Name()] = source.Statistics()
137 | }
138 | return stats
139 | }
140 |
--------------------------------------------------------------------------------
/v2/pkg/passive/sources_test.go:
--------------------------------------------------------------------------------
1 | package passive
2 |
3 | import (
4 | "fmt"
5 | "strconv"
6 | "testing"
7 |
8 | "github.com/stretchr/testify/assert"
9 | "golang.org/x/exp/maps"
10 | )
11 |
12 | var (
13 | expectedAllSources = []string{
14 | "alienvault",
15 | "anubis",
16 | "bevigil",
17 | "bufferover",
18 | "c99",
19 | "censys",
20 | "certspotter",
21 | "chaos",
22 | "chinaz",
23 | "commoncrawl",
24 | "crtsh",
25 | "digitorus",
26 | "dnsdumpster",
27 | "dnsdb",
28 | "dnsrepo",
29 | "fofa",
30 | "fullhunt",
31 | "github",
32 | "hackertarget",
33 | "intelx",
34 | "netlas",
35 | "quake",
36 | "pugrecon",
37 | "rapiddns",
38 | "redhuntlabs",
39 | // "riddler", // failing due to cloudfront protection
40 | "robtex",
41 | "rsecloud",
42 | "securitytrails",
43 | "shodan",
44 | "sitedossier",
45 | "threatbook",
46 | "threatcrowd",
47 | "virustotal",
48 | "waybackarchive",
49 | "whoisxmlapi",
50 | "zoomeyeapi",
51 | "hunter",
52 | "leakix",
53 | "facebook",
54 | // "threatminer",
55 | // "reconcloud",
56 | "builtwith",
57 | "hudsonrock",
58 | "digitalyama",
59 | }
60 |
61 | expectedDefaultSources = []string{
62 | "alienvault",
63 | "anubis",
64 | "bevigil",
65 | "bufferover",
66 | "c99",
67 | "certspotter",
68 | "censys",
69 | "chaos",
70 | "chinaz",
71 | "crtsh",
72 | "digitorus",
73 | "dnsdumpster",
74 | "dnsrepo",
75 | "fofa",
76 | "fullhunt",
77 | "hackertarget",
78 | "intelx",
79 | "quake",
80 | "redhuntlabs",
81 | "robtex",
82 | // "riddler", // failing due to cloudfront protection
83 | "rsecloud",
84 | "securitytrails",
85 | "shodan",
86 | "virustotal",
87 | "whoisxmlapi",
88 | "hunter",
89 | "leakix",
90 | "facebook",
91 | // "threatminer",
92 | // "reconcloud",
93 | "builtwith",
94 | "digitalyama",
95 | }
96 |
97 | expectedDefaultRecursiveSources = []string{
98 | "alienvault",
99 | "bufferover",
100 | "certspotter",
101 | "crtsh",
102 | "dnsdb",
103 | "digitorus",
104 | "hackertarget",
105 | "securitytrails",
106 | "virustotal",
107 | "leakix",
108 | "facebook",
109 | // "reconcloud",
110 | }
111 | )
112 |
113 | func TestSourceCategorization(t *testing.T) {
114 | defaultSources := make([]string, 0, len(AllSources))
115 | recursiveSources := make([]string, 0, len(AllSources))
116 | for _, source := range AllSources {
117 | sourceName := source.Name()
118 | if source.IsDefault() {
119 | defaultSources = append(defaultSources, sourceName)
120 | }
121 |
122 | if source.HasRecursiveSupport() {
123 | recursiveSources = append(recursiveSources, sourceName)
124 | }
125 | }
126 |
127 | assert.ElementsMatch(t, expectedDefaultSources, defaultSources)
128 | assert.ElementsMatch(t, expectedDefaultRecursiveSources, recursiveSources)
129 | assert.ElementsMatch(t, expectedAllSources, maps.Keys(NameSourceMap))
130 | }
131 |
132 | // Review: not sure if this test is necessary/useful
133 | // implementation is straightforward where sources are stored in maps and filtered based on options
134 | // the test is just checking if the filtering works as expected using count of sources
135 | func TestSourceFiltering(t *testing.T) {
136 | someSources := []string{
137 | "alienvault",
138 | "chaos",
139 | "crtsh",
140 | "virustotal",
141 | }
142 |
143 | someExclusions := []string{
144 | "alienvault",
145 | "virustotal",
146 | }
147 |
148 | tests := []struct {
149 | sources []string
150 | exclusions []string
151 | withAllSources bool
152 | withRecursion bool
153 | expectedLength int
154 | }{
155 | {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)},
156 | {someSources, someExclusions, false, true, 1},
157 | {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)},
158 |
159 | {someSources, []string{}, false, false, len(someSources)},
160 | {someSources, []string{}, true, false, len(AllSources)},
161 |
162 | {[]string{}, []string{}, false, false, len(expectedDefaultSources)},
163 | {[]string{}, []string{}, true, false, len(AllSources)},
164 | {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)},
165 | }
166 | for index, test := range tests {
167 | t.Run(strconv.Itoa(index+1), func(t *testing.T) {
168 | agent := New(test.sources, test.exclusions, test.withAllSources, test.withRecursion)
169 |
170 | for _, v := range agent.sources {
171 | fmt.Println(v.Name())
172 | }
173 |
174 | assert.Equal(t, test.expectedLength, len(agent.sources))
175 | agent = nil
176 | })
177 | }
178 | }
179 |
--------------------------------------------------------------------------------
/v2/pkg/passive/sources_w_auth_test.go:
--------------------------------------------------------------------------------
1 | package passive
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "math"
7 | "os"
8 | "reflect"
9 | "strings"
10 | "testing"
11 | "time"
12 |
13 | "github.com/stretchr/testify/assert"
14 |
15 | "github.com/projectdiscovery/gologger"
16 | "github.com/projectdiscovery/gologger/levels"
17 | "github.com/projectdiscovery/ratelimit"
18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
19 | )
20 |
21 | func TestSourcesWithKeys(t *testing.T) {
22 | if testing.Short() {
23 | t.Skip("skipping test in short mode.")
24 | }
25 |
26 | domain := "hackerone.com"
27 | timeout := 60
28 |
29 | gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
30 |
31 | ctxParent := context.Background()
32 | var multiRateLimiter *ratelimit.MultiLimiter
33 | for _, source := range AllSources {
34 | if !source.NeedsKey() {
35 | continue
36 | }
37 | multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond)
38 | }
39 |
40 | session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout)
41 | assert.Nil(t, err)
42 |
43 | var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil}
44 |
45 | for _, source := range AllSources {
46 | if !source.NeedsKey() {
47 | continue
48 | }
49 |
50 | var apiKey string
51 | if source.Name() == "chaos" {
52 | apiKey = os.Getenv("PDCP_API_KEY")
53 | } else {
54 | apiKey = os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name())))
55 | }
56 | if apiKey == "" {
57 | fmt.Printf("Skipping %s as no API key is provided\n", source.Name())
58 | continue
59 | }
60 | source.AddApiKeys([]string{apiKey})
61 |
62 | t.Run(source.Name(), func(t *testing.T) {
63 | var results []subscraping.Result
64 |
65 | ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name())
66 | for result := range source.Run(ctxWithValue, domain, session) {
67 | results = append(results, result)
68 |
69 | assert.Equal(t, source.Name(), result.Source, "wrong source name")
70 |
71 | if result.Type != subscraping.Error {
72 | assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)),
73 | fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value))
74 | } else {
75 | assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error))
76 | }
77 | }
78 |
79 | assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name()))
80 | })
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/v2/pkg/passive/sources_wo_auth_test.go:
--------------------------------------------------------------------------------
1 | package passive
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "math"
7 | "reflect"
8 | "strings"
9 | "testing"
10 | "time"
11 |
12 | "github.com/stretchr/testify/assert"
13 | "golang.org/x/exp/slices"
14 |
15 | "github.com/projectdiscovery/gologger"
16 | "github.com/projectdiscovery/gologger/levels"
17 | "github.com/projectdiscovery/ratelimit"
18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
19 | )
20 |
21 | func TestSourcesWithoutKeys(t *testing.T) {
22 | if testing.Short() {
23 | t.Skip("skipping test in short mode.")
24 | }
25 |
26 | ignoredSources := []string{
27 | "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
28 | "riddler", // failing due to cloudfront protection
29 | "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout.
30 | "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally
31 | "waybackarchive", // Fails randomly
32 | "alienvault", // 503 Service Temporarily Unavailable
33 | "digitorus", // failing with "Failed to retrieve certificate"
34 | "dnsdumpster", // failing with "unexpected status code 403 received"
35 | "anubis", // failing with "too many redirects"
36 | }
37 |
38 | domain := "hackerone.com"
39 | timeout := 60
40 |
41 | gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
42 |
43 | ctxParent := context.Background()
44 |
45 | var multiRateLimiter *ratelimit.MultiLimiter
46 | for _, source := range AllSources {
47 | if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) {
48 | continue
49 | }
50 | multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond)
51 | }
52 |
53 | session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout)
54 | assert.Nil(t, err)
55 |
56 | var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil}
57 |
58 | for _, source := range AllSources {
59 | if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) {
60 | continue
61 | }
62 |
63 | t.Run(source.Name(), func(t *testing.T) {
64 | var results []subscraping.Result
65 |
66 | ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name())
67 | for result := range source.Run(ctxWithValue, domain, session) {
68 | results = append(results, result)
69 |
70 | assert.Equal(t, source.Name(), result.Source, "wrong source name")
71 |
72 | if result.Type != subscraping.Error {
73 | assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)),
74 | fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value))
75 | } else {
76 | assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error))
77 | }
78 | }
79 |
80 | assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name()))
81 | })
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/v2/pkg/resolve/client.go:
--------------------------------------------------------------------------------
1 | package resolve
2 |
3 | import (
4 | "github.com/projectdiscovery/dnsx/libs/dnsx"
5 | )
6 |
7 | // DefaultResolvers contains the default list of resolvers known to be good
8 | var DefaultResolvers = []string{
9 | "1.1.1.1:53", // Cloudflare primary
10 | "1.0.0.1:53", // Cloudflare secondary
11 | "8.8.8.8:53", // Google primary
12 | "8.8.4.4:53", // Google secondary
13 | "9.9.9.9:53", // Quad9 Primary
14 | "9.9.9.10:53", // Quad9 Secondary
15 | "77.88.8.8:53", // Yandex Primary
16 | "77.88.8.1:53", // Yandex Secondary
17 | "208.67.222.222:53", // OpenDNS Primary
18 | "208.67.220.220:53", // OpenDNS Secondary
19 | }
20 |
21 | // Resolver is a struct for resolving DNS names
22 | type Resolver struct {
23 | DNSClient *dnsx.DNSX
24 | Resolvers []string
25 | }
26 |
27 | // New creates a new resolver struct with the default resolvers
28 | func New() *Resolver {
29 | return &Resolver{
30 | Resolvers: []string{},
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/v2/pkg/resolve/doc.go:
--------------------------------------------------------------------------------
1 | // Package resolve is used to handle resolving records
2 | // It also handles wildcard subdomains and rotating resolvers.
3 | package resolve
4 |
--------------------------------------------------------------------------------
/v2/pkg/resolve/resolve.go:
--------------------------------------------------------------------------------
1 | package resolve
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 |
7 | "github.com/rs/xid"
8 | )
9 |
10 | const (
11 | maxWildcardChecks = 3
12 | )
13 |
14 | // ResolutionPool is a pool of resolvers created for resolving subdomains
15 | // for a given host.
16 | type ResolutionPool struct {
17 | *Resolver
18 | Tasks chan HostEntry
19 | Results chan Result
20 | wg *sync.WaitGroup
21 | removeWildcard bool
22 |
23 | wildcardIPs map[string]struct{}
24 | }
25 |
26 | // HostEntry defines a host with the source
27 | type HostEntry struct {
28 | Domain string
29 | Host string
30 | Source string
31 | }
32 |
33 | // Result contains the result for a host resolution
34 | type Result struct {
35 | Type ResultType
36 | Host string
37 | IP string
38 | Error error
39 | Source string
40 | }
41 |
42 | // ResultType is the type of result found
43 | type ResultType int
44 |
45 | // Types of data result can return
46 | const (
47 | Subdomain ResultType = iota
48 | Error
49 | )
50 |
51 | // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain
52 | func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool {
53 | resolutionPool := &ResolutionPool{
54 | Resolver: r,
55 | Tasks: make(chan HostEntry),
56 | Results: make(chan Result),
57 | wg: &sync.WaitGroup{},
58 | removeWildcard: removeWildcard,
59 | wildcardIPs: make(map[string]struct{}),
60 | }
61 |
62 | go func() {
63 | for range workers {
64 | resolutionPool.wg.Add(1)
65 | go resolutionPool.resolveWorker()
66 | }
67 | resolutionPool.wg.Wait()
68 | close(resolutionPool.Results)
69 | }()
70 |
71 | return resolutionPool
72 | }
73 |
74 | // InitWildcards inits the wildcard ips array
75 | func (r *ResolutionPool) InitWildcards(domain string) error {
76 | for range maxWildcardChecks {
77 | uid := xid.New().String()
78 |
79 | hosts, _ := r.DNSClient.Lookup(uid + "." + domain)
80 | if len(hosts) == 0 {
81 | return fmt.Errorf("%s is not a wildcard domain", domain)
82 | }
83 |
84 | // Append all wildcard ips found for domains
85 | for _, host := range hosts {
86 | r.wildcardIPs[host] = struct{}{}
87 | }
88 | }
89 | return nil
90 | }
91 |
92 | func (r *ResolutionPool) resolveWorker() {
93 | for task := range r.Tasks {
94 | if !r.removeWildcard {
95 | r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source}
96 | continue
97 | }
98 |
99 | hosts, err := r.DNSClient.Lookup(task.Host)
100 | if err != nil {
101 | r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err}
102 | continue
103 | }
104 |
105 | if len(hosts) == 0 {
106 | continue
107 | }
108 |
109 | var skip bool
110 | for _, host := range hosts {
111 | // Ignore the host if it exists in wildcard ips map
112 | if _, ok := r.wildcardIPs[host]; ok {
113 | skip = true
114 | break
115 | }
116 | }
117 |
118 | if !skip {
119 | r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
120 | }
121 | }
122 | r.wg.Done()
123 | }
124 |
--------------------------------------------------------------------------------
/v2/pkg/runner/banners.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | "github.com/projectdiscovery/gologger"
5 | updateutils "github.com/projectdiscovery/utils/update"
6 | )
7 |
8 | const banner = `
9 | __ _____ __
10 | _______ __/ /_ / __(_)___ ____/ /__ _____
11 | / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/
12 | (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ /
13 | /____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/
14 | `
15 |
16 | // Name
17 | const ToolName = `subfinder`
18 |
19 | // Version is the current version of subfinder
20 | const version = `v2.8.0`
21 |
22 | // showBanner is used to show the banner to the user
23 | func showBanner() {
24 | gologger.Print().Msgf("%s\n", banner)
25 | gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
26 | }
27 |
28 | // GetUpdateCallback returns a callback function that updates subfinder
29 | func GetUpdateCallback() func() {
30 | return func() {
31 | showBanner()
32 | updateutils.GetUpdateToolCallback("subfinder", version)()
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/v2/pkg/runner/config.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | "os"
5 | "strings"
6 |
7 | "gopkg.in/yaml.v3"
8 |
9 | "github.com/projectdiscovery/gologger"
10 | "github.com/projectdiscovery/subfinder/v2/pkg/passive"
11 | fileutil "github.com/projectdiscovery/utils/file"
12 | )
13 |
14 | // createProviderConfigYAML marshals the input map to the given location on the disk
15 | func createProviderConfigYAML(configFilePath string) error {
16 | configFile, err := os.Create(configFilePath)
17 | if err != nil {
18 | return err
19 | }
20 | defer func() {
21 | if err := configFile.Close(); err != nil {
22 | gologger.Error().Msgf("Error closing config file: %s", err)
23 | }
24 | }()
25 |
26 | sourcesRequiringApiKeysMap := make(map[string][]string)
27 | for _, source := range passive.AllSources {
28 | if source.NeedsKey() {
29 | sourceName := strings.ToLower(source.Name())
30 | sourcesRequiringApiKeysMap[sourceName] = []string{}
31 | }
32 | }
33 |
34 | return yaml.NewEncoder(configFile).Encode(sourcesRequiringApiKeysMap)
35 | }
36 |
37 | // UnmarshalFrom writes the marshaled yaml config to disk
38 | func UnmarshalFrom(file string) error {
39 | reader, err := fileutil.SubstituteConfigFromEnvVars(file)
40 | if err != nil {
41 | return err
42 | }
43 |
44 | sourceApiKeysMap := map[string][]string{}
45 | err = yaml.NewDecoder(reader).Decode(sourceApiKeysMap)
46 | for _, source := range passive.AllSources {
47 | sourceName := strings.ToLower(source.Name())
48 | apiKeys := sourceApiKeysMap[sourceName]
49 | if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 {
50 | gologger.Debug().Msgf("API key(s) found for %s.", sourceName)
51 | source.AddApiKeys(apiKeys)
52 | }
53 | }
54 | return err
55 | }
56 |
--------------------------------------------------------------------------------
/v2/pkg/runner/doc.go:
--------------------------------------------------------------------------------
1 | // Package runner implements the mechanism to drive the
2 | // subdomain enumeration process
3 | package runner
4 |
--------------------------------------------------------------------------------
/v2/pkg/runner/enumerate_test.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | "os"
5 | "testing"
6 |
7 | "github.com/stretchr/testify/require"
8 | )
9 |
10 | func TestFilterAndMatchSubdomain(t *testing.T) {
11 | options := &Options{}
12 | options.Domain = []string{"example.com"}
13 | options.Threads = 10
14 | options.Timeout = 10
15 | options.Output = os.Stdout
16 | t.Run("Literal Match", func(t *testing.T) {
17 | options.Match = []string{"req.example.com"}
18 | err := options.validateOptions()
19 | if err != nil {
20 | t.Fatalf("Expected nil got %v while validation\n", err)
21 | }
22 | runner, err := NewRunner(options)
23 | if err != nil {
24 | t.Fatalf("Expected nil got %v while creating runner\n", err)
25 | }
26 | match := runner.filterAndMatchSubdomain("req.example.com")
27 | require.True(t, match, "Expecting a boolean True value ")
28 | })
29 | t.Run("Multiple Wildcards Match", func(t *testing.T) {
30 | options.Match = []string{"*.ns.*.com"}
31 | err := options.validateOptions()
32 | if err != nil {
33 | t.Fatalf("Expected nil got %v while validation\n", err)
34 | }
35 | runner, err := NewRunner(options)
36 | if err != nil {
37 | t.Fatalf("Expected nil got %v while creating runner\n", err)
38 | }
39 | subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"}
40 | for _, sub := range subdomain {
41 | match := runner.filterAndMatchSubdomain(sub)
42 | require.True(t, match, "Expecting a boolean True value ")
43 | }
44 | })
45 | t.Run("Sequential Match", func(t *testing.T) {
46 | options.Match = []string{"*.ns.example.com", "*.hackerone.com"}
47 | err := options.validateOptions()
48 | if err != nil {
49 | t.Fatalf("Expected nil got %v while validation\n", err)
50 | }
51 | runner, err := NewRunner(options)
52 | if err != nil {
53 | t.Fatalf("Expected nil got %v while creating runner\n", err)
54 | }
55 | subdomain := []string{"a.ns.example.com", "b.hackerone.com"}
56 | for _, sub := range subdomain {
57 | match := runner.filterAndMatchSubdomain(sub)
58 | require.True(t, match, "Expecting a boolean True value ")
59 | }
60 | })
61 | t.Run("Literal Filter", func(t *testing.T) {
62 | options.Filter = []string{"req.example.com"}
63 | err := options.validateOptions()
64 | if err != nil {
65 | t.Fatalf("Expected nil got %v while validation\n", err)
66 | }
67 | runner, err := NewRunner(options)
68 | if err != nil {
69 | t.Fatalf("Expected nil got %v while creating runner\n", err)
70 | }
71 | match := runner.filterAndMatchSubdomain("req.example.com")
72 | require.False(t, match, "Expecting a boolean False value ")
73 | })
74 | t.Run("Multiple Wildcards Filter", func(t *testing.T) {
75 | options.Filter = []string{"*.ns.*.com"}
76 | err := options.validateOptions()
77 | if err != nil {
78 | t.Fatalf("Expected nil got %v while validation\n", err)
79 | }
80 | runner, err := NewRunner(options)
81 | if err != nil {
82 | t.Fatalf("Expected nil got %v while creating runner\n", err)
83 | }
84 | subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"}
85 | for _, sub := range subdomain {
86 | match := runner.filterAndMatchSubdomain(sub)
87 | require.False(t, match, "Expecting a boolean False value ")
88 | }
89 | })
90 | t.Run("Sequential Filter", func(t *testing.T) {
91 | options.Filter = []string{"*.ns.example.com", "*.hackerone.com"}
92 | err := options.validateOptions()
93 | if err != nil {
94 | t.Fatalf("Expected nil got %v while validation\n", err)
95 | }
96 | runner, err := NewRunner(options)
97 | if err != nil {
98 | t.Fatalf("Expected nil got %v while creating runner\n", err)
99 | }
100 | subdomain := []string{"a.ns.example.com", "b.hackerone.com"}
101 | for _, sub := range subdomain {
102 | match := runner.filterAndMatchSubdomain(sub)
103 | require.False(t, match, "Expecting a boolean False value ")
104 | }
105 | })
106 | t.Run("Filter and Match", func(t *testing.T) {
107 | options.Filter = []string{"example.com"}
108 | options.Match = []string{"hackerone.com"}
109 | err := options.validateOptions()
110 | if err != nil {
111 | t.Fatalf("Expected nil got %v while validation\n", err)
112 | }
113 | runner, err := NewRunner(options)
114 | if err != nil {
115 | t.Fatalf("Expected nil got %v while creating runner\n", err)
116 | }
117 | subdomain := []string{"example.com", "example.com"}
118 | for _, sub := range subdomain {
119 | match := runner.filterAndMatchSubdomain(sub)
120 | require.False(t, match, "Expecting a boolean False value ")
121 | }
122 | })
123 |
124 | t.Run("Filter and Match - Same Root Domain", func(t *testing.T) {
125 | options.Filter = []string{"example.com"}
126 | options.Match = []string{"www.example.com"}
127 | err := options.validateOptions()
128 | if err != nil {
129 | t.Fatalf("Expected nil got %v while validation\n", err)
130 | }
131 | runner, err := NewRunner(options)
132 | if err != nil {
133 | t.Fatalf("Expected nil got %v while creating runner\n", err)
134 | }
135 | subdomain := map[string]string{"filter": "example.com", "match": "www.example.com"}
136 | for key, sub := range subdomain {
137 | result := runner.filterAndMatchSubdomain(sub)
138 | if key == "filter" {
139 | require.False(t, result, "Expecting a boolean False value ")
140 | } else {
141 | require.True(t, result, "Expecting a boolean True value ")
142 | }
143 | }
144 | })
145 | }
146 |
--------------------------------------------------------------------------------
/v2/pkg/runner/initialize.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | "net"
5 | "strings"
6 |
7 | "github.com/projectdiscovery/dnsx/libs/dnsx"
8 | "github.com/projectdiscovery/subfinder/v2/pkg/passive"
9 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve"
10 | )
11 |
12 | // initializePassiveEngine creates the passive engine and loads sources etc
13 | func (r *Runner) initializePassiveEngine() {
14 | r.passiveAgent = passive.New(r.options.Sources, r.options.ExcludeSources, r.options.All, r.options.OnlyRecursive)
15 | }
16 |
17 | // initializeResolver creates the resolver used to resolve the found subdomains
18 | func (r *Runner) initializeResolver() error {
19 | var resolvers []string
20 |
21 | // If the file has been provided, read resolvers from the file
22 | if r.options.ResolverList != "" {
23 | var err error
24 | resolvers, err = loadFromFile(r.options.ResolverList)
25 | if err != nil {
26 | return err
27 | }
28 | }
29 |
30 | if len(r.options.Resolvers) > 0 {
31 | resolvers = append(resolvers, r.options.Resolvers...)
32 | } else {
33 | resolvers = append(resolvers, resolve.DefaultResolvers...)
34 | }
35 |
36 | // Add default 53 UDP port if missing
37 | for i, resolver := range resolvers {
38 | if !strings.Contains(resolver, ":") {
39 | resolvers[i] = net.JoinHostPort(resolver, "53")
40 | }
41 | }
42 |
43 | r.resolverClient = resolve.New()
44 | var err error
45 | r.resolverClient.DNSClient, err = dnsx.New(dnsx.Options{BaseResolvers: resolvers, MaxRetries: 5})
46 | if err != nil {
47 | return nil
48 | }
49 |
50 | return nil
51 | }
52 |
--------------------------------------------------------------------------------
/v2/pkg/runner/stats.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | "fmt"
5 | "sort"
6 | "strings"
7 | "time"
8 |
9 | "github.com/projectdiscovery/gologger"
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | "golang.org/x/exp/maps"
12 | )
13 |
14 | func printStatistics(stats map[string]subscraping.Statistics) {
15 |
16 | sources := maps.Keys(stats)
17 | sort.Strings(sources)
18 |
19 | var lines []string
20 | var skipped []string
21 |
22 | for _, source := range sources {
23 | sourceStats := stats[source]
24 | if sourceStats.Skipped {
25 | skipped = append(skipped, fmt.Sprintf(" %s", source))
26 | } else {
27 | lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors))
28 | }
29 | }
30 |
31 | if len(lines) > 0 {
32 | gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56))
33 | gologger.Print().Msg(strings.Join(lines, "\n"))
34 | gologger.Print().Msgf("\n")
35 | }
36 |
37 | if len(skipped) > 0 {
38 | gologger.Print().Msgf("\n The following sources were included but skipped...\n\n")
39 | gologger.Print().Msg(strings.Join(skipped, "\n"))
40 | gologger.Print().Msgf("\n\n")
41 | }
42 | }
43 |
44 | func (r *Runner) GetStatistics() map[string]subscraping.Statistics {
45 | return r.passiveAgent.GetStatistics()
46 | }
47 |
--------------------------------------------------------------------------------
/v2/pkg/runner/util.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | fileutil "github.com/projectdiscovery/utils/file"
5 | stringsutil "github.com/projectdiscovery/utils/strings"
6 | )
7 |
8 | func loadFromFile(file string) ([]string, error) {
9 | chanItems, err := fileutil.ReadFile(file)
10 | if err != nil {
11 | return nil, err
12 | }
13 | var items []string
14 | for item := range chanItems {
15 | item = preprocessDomain(item)
16 | if item == "" {
17 | continue
18 | }
19 | items = append(items, item)
20 | }
21 | return items, nil
22 | }
23 |
24 | func preprocessDomain(s string) string {
25 | return stringsutil.NormalizeWithOptions(s,
26 | stringsutil.NormalizeOptions{
27 | StripComments: true,
28 | TrimCutset: "\n\t\"'` ",
29 | Lowercase: true,
30 | },
31 | )
32 | }
33 |
--------------------------------------------------------------------------------
/v2/pkg/runner/validate.go:
--------------------------------------------------------------------------------
1 | package runner
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "regexp"
7 | "strings"
8 |
9 | "github.com/projectdiscovery/gologger"
10 | "github.com/projectdiscovery/gologger/formatter"
11 | "github.com/projectdiscovery/gologger/levels"
12 | "github.com/projectdiscovery/subfinder/v2/pkg/passive"
13 | mapsutil "github.com/projectdiscovery/utils/maps"
14 | sliceutil "github.com/projectdiscovery/utils/slice"
15 | )
16 |
17 | // validateOptions validates the configuration options passed
18 | func (options *Options) validateOptions() error {
19 | // Check if domain, list of domains, or stdin info was provided.
20 | // If none was provided, then return.
21 | if len(options.Domain) == 0 && options.DomainsFile == "" && !options.Stdin {
22 | return errors.New("no input list provided")
23 | }
24 |
25 | // Both verbose and silent flags were used
26 | if options.Verbose && options.Silent {
27 | return errors.New("both verbose and silent mode specified")
28 | }
29 |
30 | // Validate threads and options
31 | if options.Threads == 0 {
32 | return errors.New("threads cannot be zero")
33 | }
34 | if options.Timeout == 0 {
35 | return errors.New("timeout cannot be zero")
36 | }
37 |
38 | // Always remove wildcard with hostip
39 | if options.HostIP && !options.RemoveWildcard {
40 | return errors.New("hostip flag must be used with RemoveWildcard option")
41 | }
42 |
43 | if options.Match != nil {
44 | options.matchRegexes = make([]*regexp.Regexp, len(options.Match))
45 | var err error
46 | for i, re := range options.Match {
47 | if options.matchRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil {
48 | return errors.New("invalid value for match regex option")
49 | }
50 | }
51 | }
52 | if options.Filter != nil {
53 | options.filterRegexes = make([]*regexp.Regexp, len(options.Filter))
54 | var err error
55 | for i, re := range options.Filter {
56 | if options.filterRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil {
57 | return errors.New("invalid value for filter regex option")
58 | }
59 | }
60 | }
61 |
62 | sources := mapsutil.GetKeys(passive.NameSourceMap)
63 | for source := range options.RateLimits.AsMap() {
64 | if !sliceutil.Contains(sources, source) {
65 | return fmt.Errorf("invalid source %s specified in -rls flag", source)
66 | }
67 | }
68 | return nil
69 | }
70 | func stripRegexString(val string) string {
71 | val = strings.ReplaceAll(val, ".", "\\.")
72 | val = strings.ReplaceAll(val, "*", ".*")
73 | return fmt.Sprint("^", val, "quot;)
74 | }
75 |
76 | // ConfigureOutput configures the output on the screen
77 | func (options *Options) ConfigureOutput() {
78 | // If the user desires verbose output, show verbose output
79 | if options.Verbose {
80 | gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
81 | }
82 | if options.NoColor {
83 | gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
84 | }
85 | if options.Silent {
86 | gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/agent.go:
--------------------------------------------------------------------------------
1 | package subscraping
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "crypto/tls"
7 | "fmt"
8 | "io"
9 | "net"
10 | "net/http"
11 | "net/url"
12 | "time"
13 |
14 | "github.com/corpix/uarand"
15 | "github.com/projectdiscovery/ratelimit"
16 |
17 | "github.com/projectdiscovery/gologger"
18 | )
19 |
20 | // NewSession creates a new session object for a domain
21 | func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLimiter, timeout int) (*Session, error) {
22 | Transport := &http.Transport{
23 | MaxIdleConns: 100,
24 | MaxIdleConnsPerHost: 100,
25 | TLSClientConfig: &tls.Config{
26 | InsecureSkipVerify: true,
27 | },
28 | Dial: (&net.Dialer{
29 | Timeout: time.Duration(timeout) * time.Second,
30 | }).Dial,
31 | }
32 |
33 | // Add proxy
34 | if proxy != "" {
35 | proxyURL, _ := url.Parse(proxy)
36 | if proxyURL == nil {
37 | // Log warning but continue anyway
38 | gologger.Warning().Msgf("Invalid proxy provided: %s", proxy)
39 | } else {
40 | Transport.Proxy = http.ProxyURL(proxyURL)
41 | }
42 | }
43 |
44 | client := &http.Client{
45 | Transport: Transport,
46 | Timeout: time.Duration(timeout) * time.Second,
47 | }
48 |
49 | session := &Session{Client: client}
50 |
51 | // Initiate rate limit instance
52 | session.MultiRateLimiter = multiRateLimiter
53 |
54 | // Create a new extractor object for the current domain
55 | extractor, err := NewSubdomainExtractor(domain)
56 | session.Extractor = extractor
57 |
58 | return session, err
59 | }
60 |
61 | // Get makes a GET request to a URL with extended parameters
62 | func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) {
63 | return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{})
64 | }
65 |
66 | // SimpleGet makes a simple GET request to a URL
67 | func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) {
68 | return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{})
69 | }
70 |
71 | // Post makes a POST request to a URL with extended parameters
72 | func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) {
73 | return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{})
74 | }
75 |
76 | // SimplePost makes a simple POST request to a URL
77 | func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) {
78 | return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{})
79 | }
80 |
81 | // HTTPRequest makes any HTTP request to a URL with extended parameters
82 | func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) {
83 | req, err := http.NewRequestWithContext(ctx, method, requestURL, body)
84 | if err != nil {
85 | return nil, err
86 | }
87 |
88 | req.Header.Set("User-Agent", uarand.GetRandom())
89 | req.Header.Set("Accept", "*/*")
90 | req.Header.Set("Accept-Language", "en")
91 | req.Header.Set("Connection", "close")
92 |
93 | if basicAuth.Username != "" || basicAuth.Password != "" {
94 | req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
95 | }
96 |
97 | if cookies != "" {
98 | req.Header.Set("Cookie", cookies)
99 | }
100 |
101 | for key, value := range headers {
102 | req.Header.Set(key, value)
103 | }
104 |
105 | sourceName := ctx.Value(CtxSourceArg).(string)
106 | mrlErr := s.MultiRateLimiter.Take(sourceName)
107 | if mrlErr != nil {
108 | return nil, mrlErr
109 | }
110 |
111 | return httpRequestWrapper(s.Client, req)
112 | }
113 |
114 | // DiscardHTTPResponse discards the response content by demand
115 | func (s *Session) DiscardHTTPResponse(response *http.Response) {
116 | if response != nil {
117 | _, err := io.Copy(io.Discard, response.Body)
118 | if err != nil {
119 | gologger.Warning().Msgf("Could not discard response body: %s\n", err)
120 | return
121 | }
122 | if closeErr := response.Body.Close(); closeErr != nil {
123 | gologger.Warning().Msgf("Could not close response body: %s\n", closeErr)
124 | }
125 | }
126 | }
127 |
128 | // Close the session
129 | func (s *Session) Close() {
130 | s.MultiRateLimiter.Stop()
131 | s.Client.CloseIdleConnections()
132 | }
133 |
134 | func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) {
135 | response, err := client.Do(request)
136 | if err != nil {
137 | return nil, err
138 | }
139 |
140 | if response.StatusCode != http.StatusOK {
141 | requestURL, _ := url.QueryUnescape(request.URL.String())
142 |
143 | gologger.Debug().MsgFunc(func() string {
144 | buffer := new(bytes.Buffer)
145 | _, _ = buffer.ReadFrom(response.Body)
146 | return fmt.Sprintf("Response for failed request against %s:\n%s", requestURL, buffer.String())
147 | })
148 | return response, fmt.Errorf("unexpected status code %d received from %s", response.StatusCode, requestURL)
149 | }
150 | return response, nil
151 | }
152 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/doc.go:
--------------------------------------------------------------------------------
1 | // Package subscraping contains the logic of scraping agents
2 | package subscraping
3 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/extractor.go:
--------------------------------------------------------------------------------
1 | package subscraping
2 |
3 | import (
4 | "regexp"
5 | "strings"
6 | )
7 |
8 | // RegexSubdomainExtractor is a concrete implementation of the SubdomainExtractor interface, using regex for extraction.
9 | type RegexSubdomainExtractor struct {
10 | extractor *regexp.Regexp
11 | }
12 |
13 | // NewSubdomainExtractor creates a new regular expression to extract
14 | // subdomains from text based on the given domain.
15 | func NewSubdomainExtractor(domain string) (*RegexSubdomainExtractor, error) {
16 | extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain)
17 | if err != nil {
18 | return nil, err
19 | }
20 | return &RegexSubdomainExtractor{extractor: extractor}, nil
21 | }
22 |
23 | // Extract implements the SubdomainExtractor interface, using the regex to find subdomains in the given text.
24 | func (re *RegexSubdomainExtractor) Extract(text string) []string {
25 | matches := re.extractor.FindAllString(text, -1)
26 | for i, match := range matches {
27 | matches[i] = strings.ToLower(match)
28 | }
29 | return matches
30 | }
31 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/alienvault/alienvault.go:
--------------------------------------------------------------------------------
1 | // Package alienvault logic
2 | package alienvault
3 |
4 | import (
5 | "context"
6 | "encoding/json"
7 | "fmt"
8 | "time"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | type alienvaultResponse struct {
14 | Detail string `json:"detail"`
15 | Error string `json:"error"`
16 | PassiveDNS []struct {
17 | Hostname string `json:"hostname"`
18 | } `json:"passive_dns"`
19 | }
20 |
21 | // Source is the passive scraping agent
22 | type Source struct {
23 | timeTaken time.Duration
24 | results int
25 | errors int
26 | }
27 |
28 | // Run function returns all subdomains found with the service
29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
30 | results := make(chan subscraping.Result)
31 | s.errors = 0
32 | s.results = 0
33 |
34 | go func() {
35 | defer func(startTime time.Time) {
36 | s.timeTaken = time.Since(startTime)
37 | close(results)
38 | }(time.Now())
39 |
40 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain))
41 | if err != nil && resp == nil {
42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
43 | s.errors++
44 | session.DiscardHTTPResponse(resp)
45 | return
46 | }
47 |
48 | var response alienvaultResponse
49 | // Get the response body and decode
50 | err = json.NewDecoder(resp.Body).Decode(&response)
51 | if err != nil {
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
53 | s.errors++
54 | session.DiscardHTTPResponse(resp)
55 | return
56 | }
57 | session.DiscardHTTPResponse(resp)
58 |
59 | if response.Error != "" {
60 | results <- subscraping.Result{
61 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error),
62 | }
63 | return
64 | }
65 |
66 | for _, record := range response.PassiveDNS {
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
68 | s.results++
69 | }
70 | }()
71 |
72 | return results
73 | }
74 |
75 | // Name returns the name of the source
76 | func (s *Source) Name() string {
77 | return "alienvault"
78 | }
79 |
80 | func (s *Source) IsDefault() bool {
81 | return true
82 | }
83 |
84 | func (s *Source) HasRecursiveSupport() bool {
85 | return true
86 | }
87 |
88 | func (s *Source) NeedsKey() bool {
89 | return false
90 | }
91 |
92 | func (s *Source) AddApiKeys(_ []string) {
93 | // no key needed
94 | }
95 |
96 | func (s *Source) Statistics() subscraping.Statistics {
97 | return subscraping.Statistics{
98 | Errors: s.errors,
99 | Results: s.results,
100 | TimeTaken: s.timeTaken,
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/anubis/anubis.go:
--------------------------------------------------------------------------------
1 | // Package anubis logic
2 | package anubis
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "net/http"
8 | "time"
9 |
10 | jsoniter "github.com/json-iterator/go"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | // Source is the passive scraping agent
16 | type Source struct {
17 | timeTaken time.Duration
18 | errors int
19 | results int
20 | }
21 |
22 | // Run function returns all subdomains found with the service
23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
24 | results := make(chan subscraping.Result)
25 | s.errors = 0
26 | s.results = 0
27 |
28 | go func() {
29 | defer func(startTime time.Time) {
30 | s.timeTaken = time.Since(startTime)
31 | close(results)
32 | }(time.Now())
33 |
34 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain))
35 | if err != nil {
36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
37 | s.errors++
38 | session.DiscardHTTPResponse(resp)
39 | return
40 | }
41 |
42 | if resp.StatusCode != http.StatusOK {
43 | session.DiscardHTTPResponse(resp)
44 | return
45 | }
46 |
47 | var subdomains []string
48 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
49 | if err != nil {
50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
51 | s.errors++
52 | session.DiscardHTTPResponse(resp)
53 | return
54 | }
55 |
56 | session.DiscardHTTPResponse(resp)
57 |
58 | for _, record := range subdomains {
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}
60 | s.results++
61 | }
62 |
63 | }()
64 |
65 | return results
66 | }
67 |
68 | // Name returns the name of the source
69 | func (s *Source) Name() string {
70 | return "anubis"
71 | }
72 |
73 | func (s *Source) IsDefault() bool {
74 | return true
75 | }
76 |
77 | func (s *Source) HasRecursiveSupport() bool {
78 | return false
79 | }
80 |
81 | func (s *Source) NeedsKey() bool {
82 | return false
83 | }
84 |
85 | func (s *Source) AddApiKeys(_ []string) {
86 | // no key needed
87 | }
88 |
89 | func (s *Source) Statistics() subscraping.Statistics {
90 | return subscraping.Statistics{
91 | Errors: s.errors,
92 | Results: s.results,
93 | TimeTaken: s.timeTaken,
94 | }
95 | }
96 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/bevigil/bevigil.go:
--------------------------------------------------------------------------------
1 | // Package bevigil logic
2 | package bevigil
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | type Response struct {
15 | Domain string `json:"domain"`
16 | Subdomains []string `json:"subdomains"`
17 | }
18 |
19 | type Source struct {
20 | apiKeys []string
21 | timeTaken time.Duration
22 | errors int
23 | results int
24 | skipped bool
25 | }
26 |
27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
28 | results := make(chan subscraping.Result)
29 | s.errors = 0
30 | s.results = 0
31 |
32 | go func() {
33 | defer func(startTime time.Time) {
34 | s.timeTaken = time.Since(startTime)
35 | close(results)
36 | }(time.Now())
37 |
38 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
39 | if randomApiKey == "" {
40 | s.skipped = true
41 | return
42 | }
43 |
44 | getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
45 |
46 | resp, err := session.Get(ctx, getUrl, "", map[string]string{
47 | "X-Access-Token": randomApiKey, "User-Agent": "subfinder",
48 | })
49 | if err != nil {
50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
51 | s.errors++
52 | session.DiscardHTTPResponse(resp)
53 | return
54 | }
55 |
56 | var subdomains []string
57 | var response Response
58 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
59 | if err != nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 | s.errors++
62 | session.DiscardHTTPResponse(resp)
63 | return
64 | }
65 |
66 | session.DiscardHTTPResponse(resp)
67 |
68 | if len(response.Subdomains) > 0 {
69 | subdomains = response.Subdomains
70 | }
71 |
72 | for _, subdomain := range subdomains {
73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
74 | s.results++
75 | }
76 |
77 | }()
78 | return results
79 | }
80 |
81 | func (s *Source) Name() string {
82 | return "bevigil"
83 | }
84 |
85 | func (s *Source) IsDefault() bool {
86 | return true
87 | }
88 |
89 | func (s *Source) HasRecursiveSupport() bool {
90 | return false
91 | }
92 |
93 | func (s *Source) NeedsKey() bool {
94 | return true
95 | }
96 |
97 | func (s *Source) AddApiKeys(keys []string) {
98 | s.apiKeys = keys
99 | }
100 |
101 | func (s *Source) Statistics() subscraping.Statistics {
102 | return subscraping.Statistics{
103 | Errors: s.errors,
104 | Results: s.results,
105 | TimeTaken: s.timeTaken,
106 | Skipped: s.skipped,
107 | }
108 | }
109 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/bufferover/bufferover.go:
--------------------------------------------------------------------------------
1 | // Package bufferover is a bufferover Scraping Engine in Golang
2 | package bufferover
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "strings"
8 | "time"
9 |
10 | jsoniter "github.com/json-iterator/go"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | type response struct {
16 | Meta struct {
17 | Errors []string `json:"Errors"`
18 | } `json:"Meta"`
19 | FDNSA []string `json:"FDNS_A"`
20 | RDNS []string `json:"RDNS"`
21 | Results []string `json:"Results"`
22 | }
23 |
24 | // Source is the passive scraping agent
25 | type Source struct {
26 | apiKeys []string
27 | timeTaken time.Duration
28 | errors int
29 | results int
30 | skipped bool
31 | }
32 |
33 | // Run function returns all subdomains found with the service
34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
35 | results := make(chan subscraping.Result)
36 | s.errors = 0
37 | s.results = 0
38 |
39 | go func() {
40 | defer func(startTime time.Time) {
41 | s.timeTaken = time.Since(startTime)
42 | close(results)
43 | }(time.Now())
44 |
45 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
46 | if randomApiKey == "" {
47 | s.skipped = true
48 | return
49 | }
50 |
51 | s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), randomApiKey, session, results)
52 | }()
53 |
54 | return results
55 | }
56 |
57 | func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) {
58 | resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey})
59 |
60 | if err != nil && resp == nil {
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
62 | s.errors++
63 | session.DiscardHTTPResponse(resp)
64 | return
65 | }
66 |
67 | var bufforesponse response
68 | err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse)
69 | if err != nil {
70 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
71 | s.errors++
72 | session.DiscardHTTPResponse(resp)
73 | return
74 | }
75 |
76 | session.DiscardHTTPResponse(resp)
77 |
78 | metaErrors := bufforesponse.Meta.Errors
79 |
80 | if len(metaErrors) > 0 {
81 | results <- subscraping.Result{
82 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", ")),
83 | }
84 | s.errors++
85 | return
86 | }
87 |
88 | var subdomains []string
89 |
90 | if len(bufforesponse.FDNSA) > 0 {
91 | subdomains = bufforesponse.FDNSA
92 | subdomains = append(subdomains, bufforesponse.RDNS...)
93 | } else if len(bufforesponse.Results) > 0 {
94 | subdomains = bufforesponse.Results
95 | }
96 |
97 | for _, subdomain := range subdomains {
98 | for _, value := range session.Extractor.Extract(subdomain) {
99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
100 | s.results++
101 | }
102 | }
103 | }
104 |
105 | // Name returns the name of the source
106 | func (s *Source) Name() string {
107 | return "bufferover"
108 | }
109 |
110 | func (s *Source) IsDefault() bool {
111 | return true
112 | }
113 |
114 | func (s *Source) HasRecursiveSupport() bool {
115 | return true
116 | }
117 |
118 | func (s *Source) NeedsKey() bool {
119 | return true
120 | }
121 |
122 | func (s *Source) AddApiKeys(keys []string) {
123 | s.apiKeys = keys
124 | }
125 |
126 | func (s *Source) Statistics() subscraping.Statistics {
127 | return subscraping.Statistics{
128 | Errors: s.errors,
129 | Results: s.results,
130 | TimeTaken: s.timeTaken,
131 | Skipped: s.skipped,
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/builtwith/builtwith.go:
--------------------------------------------------------------------------------
1 | // Package builtwith logic
2 | package builtwith
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | type response struct {
15 | Results []resultItem `json:"Results"`
16 | }
17 |
18 | type resultItem struct {
19 | Result result `json:"Result"`
20 | }
21 |
22 | type result struct {
23 | Paths []path `json:"Paths"`
24 | }
25 |
26 | type path struct {
27 | Domain string `json:"Domain"`
28 | Url string `json:"Url"`
29 | SubDomain string `json:"SubDomain"`
30 | }
31 |
32 | // Source is the passive scraping agent
33 | type Source struct {
34 | apiKeys []string
35 | timeTaken time.Duration
36 | errors int
37 | results int
38 | skipped bool
39 | }
40 |
41 | // Run function returns all subdomains found with the service
42 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
43 | results := make(chan subscraping.Result)
44 | s.errors = 0
45 | s.results = 0
46 |
47 | go func() {
48 | defer func(startTime time.Time) {
49 | s.timeTaken = time.Since(startTime)
50 | close(results)
51 | }(time.Now())
52 |
53 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
54 | if randomApiKey == "" {
55 | return
56 | }
57 |
58 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain))
59 | if err != nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 | s.errors++
62 | session.DiscardHTTPResponse(resp)
63 | return
64 | }
65 |
66 | var data response
67 | err = jsoniter.NewDecoder(resp.Body).Decode(&data)
68 | if err != nil {
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
70 | s.errors++
71 | session.DiscardHTTPResponse(resp)
72 | return
73 | }
74 | session.DiscardHTTPResponse(resp)
75 | for _, result := range data.Results {
76 | for _, path := range result.Result.Paths {
77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}
78 | s.results++
79 | }
80 | }
81 | }()
82 |
83 | return results
84 | }
85 |
86 | // Name returns the name of the source
87 | func (s *Source) Name() string {
88 | return "builtwith"
89 | }
90 |
91 | func (s *Source) IsDefault() bool {
92 | return true
93 | }
94 |
95 | func (s *Source) HasRecursiveSupport() bool {
96 | return false
97 | }
98 |
99 | func (s *Source) NeedsKey() bool {
100 | return true
101 | }
102 |
103 | func (s *Source) AddApiKeys(keys []string) {
104 | s.apiKeys = keys
105 | }
106 |
107 | func (s *Source) Statistics() subscraping.Statistics {
108 | return subscraping.Statistics{
109 | Errors: s.errors,
110 | Results: s.results,
111 | TimeTaken: s.timeTaken,
112 | Skipped: s.skipped,
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/c99/c99.go:
--------------------------------------------------------------------------------
1 | // Package c99 logic
2 | package c99
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "strings"
8 | "time"
9 |
10 | jsoniter "github.com/json-iterator/go"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | // Source is the passive scraping agent
16 | type Source struct {
17 | apiKeys []string
18 | timeTaken time.Duration
19 | errors int
20 | results int
21 | skipped bool
22 | }
23 |
24 | type dnsdbLookupResponse struct {
25 | Success bool `json:"success"`
26 | Subdomains []struct {
27 | Subdomain string `json:"subdomain"`
28 | IP string `json:"ip"`
29 | Cloudflare bool `json:"cloudflare"`
30 | } `json:"subdomains"`
31 | Error string `json:"error"`
32 | }
33 |
34 | // Run function returns all subdomains found with the service
35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
36 | results := make(chan subscraping.Result)
37 | s.errors = 0
38 | s.results = 0
39 |
40 | go func() {
41 | defer func(startTime time.Time) {
42 | s.timeTaken = time.Since(startTime)
43 | close(results)
44 | }(time.Now())
45 |
46 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
47 | if randomApiKey == "" {
48 | s.skipped = true
49 | return
50 | }
51 |
52 | searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain)
53 | resp, err := session.SimpleGet(ctx, searchURL)
54 | if err != nil {
55 | session.DiscardHTTPResponse(resp)
56 | return
57 | }
58 |
59 | defer func() {
60 | if err := resp.Body.Close(); err != nil {
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
62 | s.errors++
63 | }
64 | }()
65 |
66 | var response dnsdbLookupResponse
67 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
68 | if err != nil {
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
70 | s.errors++
71 | return
72 | }
73 |
74 | if response.Error != "" {
75 | results <- subscraping.Result{
76 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error),
77 | }
78 | s.errors++
79 | return
80 | }
81 |
82 | for _, data := range response.Subdomains {
83 | if !strings.HasPrefix(data.Subdomain, ".") {
84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}
85 | s.results++
86 | }
87 | }
88 | }()
89 |
90 | return results
91 | }
92 |
93 | // Name returns the name of the source
94 | func (s *Source) Name() string {
95 | return "c99"
96 | }
97 |
98 | func (s *Source) IsDefault() bool {
99 | return true
100 | }
101 |
102 | func (s *Source) HasRecursiveSupport() bool {
103 | return false
104 | }
105 |
106 | func (s *Source) NeedsKey() bool {
107 | return true
108 | }
109 |
110 | func (s *Source) AddApiKeys(keys []string) {
111 | s.apiKeys = keys
112 | }
113 |
114 | func (s *Source) Statistics() subscraping.Statistics {
115 | return subscraping.Statistics{
116 | Errors: s.errors,
117 | Results: s.results,
118 | TimeTaken: s.timeTaken,
119 | Skipped: s.skipped,
120 | }
121 | }
122 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/censys/censys.go:
--------------------------------------------------------------------------------
1 | // Package censys logic
2 | package censys
3 |
4 | import (
5 | "context"
6 | "strconv"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | urlutil "github.com/projectdiscovery/utils/url"
13 | )
14 |
15 | const (
16 | maxCensysPages = 10
17 | maxPerPage = 100
18 | )
19 |
20 | type response struct {
21 | Code int `json:"code"`
22 | Status string `json:"status"`
23 | Result result `json:"result"`
24 | }
25 |
26 | type result struct {
27 | Query string `json:"query"`
28 | Total float64 `json:"total"`
29 | DurationMS int `json:"duration_ms"`
30 | Hits []hit `json:"hits"`
31 | Links links `json:"links"`
32 | }
33 |
34 | type hit struct {
35 | Parsed parsed `json:"parsed"`
36 | Names []string `json:"names"`
37 | FingerprintSha256 string `json:"fingerprint_sha256"`
38 | }
39 |
40 | type parsed struct {
41 | ValidityPeriod validityPeriod `json:"validity_period"`
42 | SubjectDN string `json:"subject_dn"`
43 | IssuerDN string `json:"issuer_dn"`
44 | }
45 |
46 | type validityPeriod struct {
47 | NotAfter string `json:"not_after"`
48 | NotBefore string `json:"not_before"`
49 | }
50 |
51 | type links struct {
52 | Next string `json:"next"`
53 | Prev string `json:"prev"`
54 | }
55 |
56 | // Source is the passive scraping agent
57 | type Source struct {
58 | apiKeys []apiKey
59 | timeTaken time.Duration
60 | errors int
61 | results int
62 | skipped bool
63 | }
64 |
65 | type apiKey struct {
66 | token string
67 | secret string
68 | }
69 |
70 | // Run function returns all subdomains found with the service
71 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
72 | results := make(chan subscraping.Result)
73 | s.errors = 0
74 | s.results = 0
75 |
76 | go func() {
77 | defer func(startTime time.Time) {
78 | s.timeTaken = time.Since(startTime)
79 | close(results)
80 | }(time.Now())
81 |
82 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
83 | if randomApiKey.token == "" || randomApiKey.secret == "" {
84 | s.skipped = true
85 | return
86 | }
87 |
88 | certSearchEndpoint := "https://search.censys.io/api/v2/certificates/search"
89 | cursor := ""
90 | currentPage := 1
91 | for {
92 | certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint)
93 | if err != nil {
94 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
95 | s.errors++
96 | return
97 | }
98 |
99 | certSearchEndpointUrl.Params.Add("q", domain)
100 | certSearchEndpointUrl.Params.Add("per_page", strconv.Itoa(maxPerPage))
101 | if cursor != "" {
102 | certSearchEndpointUrl.Params.Add("cursor", cursor)
103 | }
104 |
105 | resp, err := session.HTTPRequest(
106 | ctx,
107 | "GET",
108 | certSearchEndpointUrl.String(),
109 | "",
110 | nil,
111 | nil,
112 | subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret},
113 | )
114 |
115 | if err != nil {
116 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
117 | s.errors++
118 | session.DiscardHTTPResponse(resp)
119 | return
120 | }
121 |
122 | var censysResponse response
123 | err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse)
124 | if err != nil {
125 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
126 | s.errors++
127 | session.DiscardHTTPResponse(resp)
128 | return
129 | }
130 |
131 | session.DiscardHTTPResponse(resp)
132 |
133 | for _, hit := range censysResponse.Result.Hits {
134 | for _, name := range hit.Names {
135 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}
136 | s.results++
137 | }
138 | }
139 |
140 | // Exit the censys enumeration if last page is reached
141 | cursor = censysResponse.Result.Links.Next
142 | if cursor == "" || currentPage >= maxCensysPages {
143 | break
144 | }
145 | currentPage++
146 | }
147 | }()
148 |
149 | return results
150 | }
151 |
152 | // Name returns the name of the source
153 | func (s *Source) Name() string {
154 | return "censys"
155 | }
156 |
157 | func (s *Source) IsDefault() bool {
158 | return true
159 | }
160 |
161 | func (s *Source) HasRecursiveSupport() bool {
162 | return false
163 | }
164 |
165 | func (s *Source) NeedsKey() bool {
166 | return true
167 | }
168 |
169 | func (s *Source) AddApiKeys(keys []string) {
170 | s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
171 | return apiKey{k, v}
172 | })
173 | }
174 |
175 | func (s *Source) Statistics() subscraping.Statistics {
176 | return subscraping.Statistics{
177 | Errors: s.errors,
178 | Results: s.results,
179 | TimeTaken: s.timeTaken,
180 | Skipped: s.skipped,
181 | }
182 | }
183 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/certspotter/certspotter.go:
--------------------------------------------------------------------------------
1 | // Package certspotter logic
2 | package certspotter
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | type certspotterObject struct {
15 | ID string `json:"id"`
16 | DNSNames []string `json:"dns_names"`
17 | }
18 |
19 | // Source is the passive scraping agent
20 | type Source struct {
21 | apiKeys []string
22 | timeTaken time.Duration
23 | errors int
24 | results int
25 | skipped bool
26 | }
27 |
28 | // Run function returns all subdomains found with the service
29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
30 | results := make(chan subscraping.Result)
31 | s.errors = 0
32 | s.results = 0
33 |
34 | go func() {
35 | defer func(startTime time.Time) {
36 | s.timeTaken = time.Since(startTime)
37 | close(results)
38 | }(time.Now())
39 |
40 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
41 | if randomApiKey == "" {
42 | s.skipped = true
43 | return
44 | }
45 |
46 | headers := map[string]string{"Authorization": "Bearer " + randomApiKey}
47 | cookies := ""
48 |
49 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers)
50 | if err != nil {
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
52 | s.errors++
53 | session.DiscardHTTPResponse(resp)
54 | return
55 | }
56 |
57 | var response []certspotterObject
58 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
59 | if err != nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 | s.errors++
62 | session.DiscardHTTPResponse(resp)
63 | return
64 | }
65 | session.DiscardHTTPResponse(resp)
66 |
67 | for _, cert := range response {
68 | for _, subdomain := range cert.DNSNames {
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
70 | s.results++
71 | }
72 | }
73 |
74 | // if the number of responses is zero, close the channel and return.
75 | if len(response) == 0 {
76 | return
77 | }
78 |
79 | id := response[len(response)-1].ID
80 | for {
81 | reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id)
82 |
83 | resp, err := session.Get(ctx, reqURL, cookies, headers)
84 | if err != nil {
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
86 | s.errors++
87 | return
88 | }
89 |
90 | var response []certspotterObject
91 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
92 | if err != nil {
93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
94 | s.errors++
95 | session.DiscardHTTPResponse(resp)
96 | return
97 | }
98 | session.DiscardHTTPResponse(resp)
99 |
100 | if len(response) == 0 {
101 | break
102 | }
103 |
104 | for _, cert := range response {
105 | for _, subdomain := range cert.DNSNames {
106 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
107 | s.results++
108 | }
109 | }
110 |
111 | id = response[len(response)-1].ID
112 | }
113 | }()
114 |
115 | return results
116 | }
117 |
118 | // Name returns the name of the source
119 | func (s *Source) Name() string {
120 | return "certspotter"
121 | }
122 |
123 | func (s *Source) IsDefault() bool {
124 | return true
125 | }
126 |
127 | func (s *Source) HasRecursiveSupport() bool {
128 | return true
129 | }
130 |
131 | func (s *Source) NeedsKey() bool {
132 | return true
133 | }
134 |
135 | func (s *Source) AddApiKeys(keys []string) {
136 | s.apiKeys = keys
137 | }
138 |
139 | func (s *Source) Statistics() subscraping.Statistics {
140 | return subscraping.Statistics{
141 | Errors: s.errors,
142 | Results: s.results,
143 | TimeTaken: s.timeTaken,
144 | Skipped: s.skipped,
145 | }
146 | }
147 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/chaos/chaos.go:
--------------------------------------------------------------------------------
1 | // Package chaos logic
2 | package chaos
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | "github.com/projectdiscovery/chaos-client/pkg/chaos"
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | // Source is the passive scraping agent
14 | type Source struct {
15 | apiKeys []string
16 | timeTaken time.Duration
17 | errors int
18 | results int
19 | skipped bool
20 | }
21 |
22 | // Run function returns all subdomains found with the service
23 | func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result {
24 | results := make(chan subscraping.Result)
25 | s.errors = 0
26 | s.results = 0
27 |
28 | go func() {
29 | defer func(startTime time.Time) {
30 | s.timeTaken = time.Since(startTime)
31 | close(results)
32 | }(time.Now())
33 |
34 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
35 | if randomApiKey == "" {
36 | s.skipped = true
37 | return
38 | }
39 |
40 | chaosClient := chaos.New(randomApiKey)
41 | for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{
42 | Domain: domain,
43 | }) {
44 | if result.Error != nil {
45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error}
46 | s.errors++
47 | break
48 | }
49 | results <- subscraping.Result{
50 | Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain),
51 | }
52 | s.results++
53 | }
54 | }()
55 |
56 | return results
57 | }
58 |
59 | // Name returns the name of the source
60 | func (s *Source) Name() string {
61 | return "chaos"
62 | }
63 |
64 | func (s *Source) IsDefault() bool {
65 | return true
66 | }
67 |
68 | func (s *Source) HasRecursiveSupport() bool {
69 | return false
70 | }
71 |
72 | func (s *Source) NeedsKey() bool {
73 | return true
74 | }
75 |
76 | func (s *Source) AddApiKeys(keys []string) {
77 | s.apiKeys = keys
78 | }
79 |
80 | func (s *Source) Statistics() subscraping.Statistics {
81 | return subscraping.Statistics{
82 | Errors: s.errors,
83 | Results: s.results,
84 | TimeTaken: s.timeTaken,
85 | Skipped: s.skipped,
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/chinaz/chinaz.go:
--------------------------------------------------------------------------------
1 | package chinaz
2 |
3 | // chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi
4 | import (
5 | "context"
6 | "fmt"
7 | "io"
8 | "time"
9 |
10 | jsoniter "github.com/json-iterator/go"
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | // Source is the passive scraping agent
15 | type Source struct {
16 | apiKeys []string
17 | timeTaken time.Duration
18 | errors int
19 | results int
20 | skipped bool
21 | }
22 |
23 | // Run function returns all subdomains found with the service
24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
25 | results := make(chan subscraping.Result)
26 | s.errors = 0
27 | s.results = 0
28 |
29 | go func() {
30 | defer func(startTime time.Time) {
31 | s.timeTaken = time.Since(startTime)
32 | close(results)
33 | }(time.Now())
34 |
35 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
36 | if randomApiKey == "" {
37 | s.skipped = true
38 | return
39 | }
40 |
41 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain))
42 | if err != nil {
43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
44 | s.errors++
45 | session.DiscardHTTPResponse(resp)
46 | return
47 | }
48 |
49 | body, err := io.ReadAll(resp.Body)
50 |
51 | session.DiscardHTTPResponse(resp)
52 |
53 | SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList")
54 |
55 | if SubdomainList.ToBool() {
56 | _data := []byte(SubdomainList.ToString())
57 | for i := 0; i < SubdomainList.Size(); i++ {
58 | subdomain := jsoniter.Get(_data, i, "DataUrl").ToString()
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
60 | s.results++
61 | }
62 | } else {
63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
64 | s.errors++
65 | return
66 | }
67 | }()
68 |
69 | return results
70 | }
71 |
72 | // Name returns the name of the source
73 | func (s *Source) Name() string {
74 | return "chinaz"
75 | }
76 |
77 | func (s *Source) IsDefault() bool {
78 | return true
79 | }
80 |
81 | func (s *Source) HasRecursiveSupport() bool {
82 | return false
83 | }
84 |
85 | func (s *Source) NeedsKey() bool {
86 | return true
87 | }
88 |
89 | func (s *Source) AddApiKeys(keys []string) {
90 | s.apiKeys = keys
91 | }
92 |
93 | func (s *Source) Statistics() subscraping.Statistics {
94 | return subscraping.Statistics{
95 | Errors: s.errors,
96 | Results: s.results,
97 | TimeTaken: s.timeTaken,
98 | Skipped: s.skipped,
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go:
--------------------------------------------------------------------------------
1 | // Package commoncrawl logic
2 | package commoncrawl
3 |
4 | import (
5 | "bufio"
6 | "context"
7 | "fmt"
8 | "net/url"
9 | "strconv"
10 | "strings"
11 | "time"
12 |
13 | jsoniter "github.com/json-iterator/go"
14 |
15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
16 | )
17 |
18 | const (
19 | indexURL = "https://index.commoncrawl.org/collinfo.json"
20 | maxYearsBack = 5
21 | )
22 |
23 | var year = time.Now().Year()
24 |
25 | type indexResponse struct {
26 | ID string `json:"id"`
27 | APIURL string `json:"cdx-api"`
28 | }
29 |
30 | // Source is the passive scraping agent
31 | type Source struct {
32 | timeTaken time.Duration
33 | errors int
34 | results int
35 | }
36 |
37 | // Run function returns all subdomains found with the service
38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
39 | results := make(chan subscraping.Result)
40 | s.errors = 0
41 | s.results = 0
42 |
43 | go func() {
44 | defer func(startTime time.Time) {
45 | s.timeTaken = time.Since(startTime)
46 | close(results)
47 | }(time.Now())
48 |
49 | resp, err := session.SimpleGet(ctx, indexURL)
50 | if err != nil {
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
52 | s.errors++
53 | session.DiscardHTTPResponse(resp)
54 | return
55 | }
56 |
57 | var indexes []indexResponse
58 | err = jsoniter.NewDecoder(resp.Body).Decode(&indexes)
59 | if err != nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 | s.errors++
62 | session.DiscardHTTPResponse(resp)
63 | return
64 | }
65 | session.DiscardHTTPResponse(resp)
66 |
67 | years := make([]string, 0)
68 | for i := range maxYearsBack {
69 | years = append(years, strconv.Itoa(year-i))
70 | }
71 |
72 | searchIndexes := make(map[string]string)
73 | for _, year := range years {
74 | for _, index := range indexes {
75 | if strings.Contains(index.ID, year) {
76 | if _, ok := searchIndexes[year]; !ok {
77 | searchIndexes[year] = index.APIURL
78 | break
79 | }
80 | }
81 | }
82 | }
83 |
84 | for _, apiURL := range searchIndexes {
85 | further := s.getSubdomains(ctx, apiURL, domain, session, results)
86 | if !further {
87 | break
88 | }
89 | }
90 | }()
91 |
92 | return results
93 | }
94 |
95 | // Name returns the name of the source
96 | func (s *Source) Name() string {
97 | return "commoncrawl"
98 | }
99 |
100 | func (s *Source) IsDefault() bool {
101 | return false
102 | }
103 |
104 | func (s *Source) HasRecursiveSupport() bool {
105 | return false
106 | }
107 |
108 | func (s *Source) NeedsKey() bool {
109 | return false
110 | }
111 |
112 | func (s *Source) AddApiKeys(_ []string) {
113 | // no key needed
114 | }
115 |
116 | func (s *Source) Statistics() subscraping.Statistics {
117 | return subscraping.Statistics{
118 | Errors: s.errors,
119 | Results: s.results,
120 | TimeTaken: s.timeTaken,
121 | }
122 | }
123 |
124 | func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
125 | for {
126 | select {
127 | case <-ctx.Done():
128 | return false
129 | default:
130 | var headers = map[string]string{"Host": "index.commoncrawl.org"}
131 | resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers)
132 | if err != nil {
133 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
134 | s.errors++
135 | session.DiscardHTTPResponse(resp)
136 | return false
137 | }
138 |
139 | scanner := bufio.NewScanner(resp.Body)
140 | for scanner.Scan() {
141 | line := scanner.Text()
142 | if line == "" {
143 | continue
144 | }
145 | line, _ = url.QueryUnescape(line)
146 | for _, subdomain := range session.Extractor.Extract(line) {
147 | if subdomain != "" {
148 | // fix for triple encoded URL
149 | subdomain = strings.ToLower(subdomain)
150 | subdomain = strings.TrimPrefix(subdomain, "25")
151 | subdomain = strings.TrimPrefix(subdomain, "2f")
152 |
153 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
154 | s.results++
155 | }
156 | }
157 | }
158 | session.DiscardHTTPResponse(resp)
159 | return true
160 | }
161 | }
162 | }
163 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/crtsh/crtsh.go:
--------------------------------------------------------------------------------
1 | // Package crtsh logic
2 | package crtsh
3 |
4 | import (
5 | "context"
6 | "database/sql"
7 | "fmt"
8 | "strconv"
9 | "strings"
10 | "time"
11 |
12 | jsoniter "github.com/json-iterator/go"
13 |
14 | // postgres driver
15 | _ "github.com/lib/pq"
16 |
17 | "github.com/projectdiscovery/gologger"
18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
19 | contextutil "github.com/projectdiscovery/utils/context"
20 | )
21 |
22 | type subdomain struct {
23 | ID int `json:"id"`
24 | NameValue string `json:"name_value"`
25 | }
26 |
27 | // Source is the passive scraping agent
28 | type Source struct {
29 | timeTaken time.Duration
30 | errors int
31 | results int
32 | }
33 |
34 | // Run function returns all subdomains found with the service
35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
36 | results := make(chan subscraping.Result)
37 | s.errors = 0
38 | s.results = 0
39 |
40 | go func() {
41 | defer func(startTime time.Time) {
42 | s.timeTaken = time.Since(startTime)
43 | close(results)
44 | }(time.Now())
45 |
46 | count := s.getSubdomainsFromSQL(ctx, domain, session, results)
47 | if count > 0 {
48 | return
49 | }
50 | _ = s.getSubdomainsFromHTTP(ctx, domain, session, results)
51 | }()
52 |
53 | return results
54 | }
55 |
56 | func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int {
57 | db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes")
58 | if err != nil {
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
60 | s.errors++
61 | return 0
62 | }
63 |
64 | defer func() {
65 | if closeErr := db.Close(); closeErr != nil {
66 | gologger.Warning().Msgf("Could not close database connection: %s\n", closeErr)
67 | }
68 | }()
69 |
70 | limitClause := ""
71 | if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok {
72 | if allBool, err := strconv.ParseBool(string(all)); err == nil && !allBool {
73 | limitClause = "LIMIT 10000"
74 | }
75 | }
76 |
77 | query := fmt.Sprintf(`WITH ci AS (
78 | SELECT min(sub.CERTIFICATE_ID) ID,
79 | min(sub.ISSUER_CA_ID) ISSUER_CA_ID,
80 | array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES,
81 | x509_commonName(sub.CERTIFICATE) COMMON_NAME,
82 | x509_notBefore(sub.CERTIFICATE) NOT_BEFORE,
83 | x509_notAfter(sub.CERTIFICATE) NOT_AFTER,
84 | encode(x509_serialNumber(sub.CERTIFICATE), 'hex') SERIAL_NUMBER
85 | FROM (SELECT *
86 | FROM certificate_and_identities cai
87 | WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)
88 | AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%')
89 | %s
90 | ) sub
91 | GROUP BY sub.CERTIFICATE
92 | )
93 | SELECT array_to_string(ci.NAME_VALUES, chr(10)) NAME_VALUE
94 | FROM ci
95 | LEFT JOIN LATERAL (
96 | SELECT min(ctle.ENTRY_TIMESTAMP) ENTRY_TIMESTAMP
97 | FROM ct_log_entry ctle
98 | WHERE ctle.CERTIFICATE_ID = ci.ID
99 | ) le ON TRUE,
100 | ca
101 | WHERE ci.ISSUER_CA_ID = ca.ID
102 | ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause)
103 | rows, err := db.QueryContext(ctx, query, domain)
104 | if err != nil {
105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
106 | s.errors++
107 | return 0
108 | }
109 | if err := rows.Err(); err != nil {
110 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
111 | s.errors++
112 | return 0
113 | }
114 |
115 | var count int
116 | var data string
117 | // Parse all the rows getting subdomains
118 | for rows.Next() {
119 | err := rows.Scan(&data)
120 | if err != nil {
121 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
122 | s.errors++
123 | return count
124 | }
125 |
126 | count++
127 | for subdomain := range strings.SplitSeq(data, "\n") {
128 | for _, value := range session.Extractor.Extract(subdomain) {
129 | if value != "" {
130 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
131 | s.results++
132 | }
133 | }
134 | }
135 | }
136 | return count
137 | }
138 |
139 | func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
140 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain))
141 | if err != nil {
142 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
143 | s.errors++
144 | session.DiscardHTTPResponse(resp)
145 | return false
146 | }
147 |
148 | var subdomains []subdomain
149 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
150 | if err != nil {
151 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
152 | s.errors++
153 | session.DiscardHTTPResponse(resp)
154 | return false
155 | }
156 |
157 | session.DiscardHTTPResponse(resp)
158 |
159 | for _, subdomain := range subdomains {
160 | for sub := range strings.SplitSeq(subdomain.NameValue, "\n") {
161 | for _, value := range session.Extractor.Extract(sub) {
162 | if value != "" {
163 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
164 | s.results++
165 | }
166 | }
167 | }
168 | }
169 |
170 | return true
171 | }
172 |
173 | // Name returns the name of the source
174 | func (s *Source) Name() string {
175 | return "crtsh"
176 | }
177 |
178 | func (s *Source) IsDefault() bool {
179 | return true
180 | }
181 |
182 | func (s *Source) HasRecursiveSupport() bool {
183 | return true
184 | }
185 |
186 | func (s *Source) NeedsKey() bool {
187 | return false
188 | }
189 |
190 | func (s *Source) AddApiKeys(_ []string) {
191 | // no key needed
192 | }
193 |
194 | func (s *Source) Statistics() subscraping.Statistics {
195 | return subscraping.Statistics{
196 | Errors: s.errors,
197 | Results: s.results,
198 | TimeTaken: s.timeTaken,
199 | }
200 | }
201 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/digitalyama/digitalyama.go:
--------------------------------------------------------------------------------
1 | package digitalyama
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "time"
7 |
8 | jsoniter "github.com/json-iterator/go"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | // Source is the passive scraping agent
14 | type Source struct {
15 | apiKeys []string
16 | timeTaken time.Duration
17 | errors int
18 | results int
19 | skipped bool
20 | }
21 |
22 | type digitalYamaResponse struct {
23 | Query string `json:"query"`
24 | Count int `json:"count"`
25 | Subdomains []string `json:"subdomains"`
26 | UsageSummary struct {
27 | QueryCost float64 `json:"query_cost"`
28 | CreditsRemaining float64 `json:"credits_remaining"`
29 | } `json:"usage_summary"`
30 | }
31 |
32 | // Run function returns all subdomains found with the service
33 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
34 | results := make(chan subscraping.Result)
35 | s.errors = 0
36 | s.results = 0
37 |
38 | go func() {
39 | defer func(startTime time.Time) {
40 | s.timeTaken = time.Since(startTime)
41 | close(results)
42 | }(time.Now())
43 |
44 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
45 | if randomApiKey == "" {
46 | s.skipped = true
47 | return
48 | }
49 |
50 | searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain)
51 | resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey})
52 | if err != nil {
53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
54 | s.errors++
55 | return
56 | }
57 | defer func() {
58 | if err := resp.Body.Close(); err != nil {
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
60 | s.errors++
61 | }
62 | }()
63 |
64 | if resp.StatusCode != 200 {
65 | var errResponse struct {
66 | Detail []struct {
67 | Loc []string `json:"loc"`
68 | Msg string `json:"msg"`
69 | Type string `json:"type"`
70 | } `json:"detail"`
71 | }
72 | err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse)
73 | if err != nil {
74 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)}
75 | s.errors++
76 | return
77 | }
78 | if len(errResponse.Detail) > 0 {
79 | errMsg := errResponse.Detail[0].Msg
80 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)}
81 | } else {
82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)}
83 | }
84 | s.errors++
85 | return
86 | }
87 |
88 | var response digitalYamaResponse
89 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
90 | if err != nil {
91 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
92 | s.errors++
93 | return
94 | }
95 |
96 | for _, subdomain := range response.Subdomains {
97 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
98 | s.results++
99 | }
100 | }()
101 |
102 | return results
103 | }
104 |
105 | // Name returns the name of the source
106 | func (s *Source) Name() string {
107 | return "digitalyama"
108 | }
109 |
110 | func (s *Source) IsDefault() bool {
111 | return true
112 | }
113 |
114 | func (s *Source) HasRecursiveSupport() bool {
115 | return false
116 | }
117 |
118 | func (s *Source) NeedsKey() bool {
119 | return true
120 | }
121 |
122 | func (s *Source) AddApiKeys(keys []string) {
123 | s.apiKeys = keys
124 | }
125 |
126 | func (s *Source) Statistics() subscraping.Statistics {
127 | return subscraping.Statistics{
128 | Errors: s.errors,
129 | Results: s.results,
130 | TimeTaken: s.timeTaken,
131 | Skipped: s.skipped,
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/digitorus/digitorus.go:
--------------------------------------------------------------------------------
1 | // Package waybackarchive logic
2 | package digitorus
3 |
4 | import (
5 | "bufio"
6 | "context"
7 | "fmt"
8 | "net/http"
9 | "strings"
10 | "time"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | "github.com/projectdiscovery/utils/ptr"
14 | )
15 |
16 | // Source is the passive scraping agent
17 | type Source struct {
18 | timeTaken time.Duration
19 | errors int
20 | results int
21 | }
22 |
23 | // Run function returns all subdomains found with the service
24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
25 | results := make(chan subscraping.Result)
26 | s.errors = 0
27 | s.results = 0
28 |
29 | go func() {
30 | defer func(startTime time.Time) {
31 | s.timeTaken = time.Since(startTime)
32 | close(results)
33 | }(time.Now())
34 |
35 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain))
36 | // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774
37 | if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound {
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
39 | s.errors++
40 | session.DiscardHTTPResponse(resp)
41 | return
42 | }
43 |
44 | defer func() {
45 | if err := resp.Body.Close(); err != nil {
46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
47 | s.errors++
48 | }
49 | }()
50 |
51 | scanner := bufio.NewScanner(resp.Body)
52 | for scanner.Scan() {
53 | line := scanner.Text()
54 | if line == "" {
55 | continue
56 | }
57 | subdomains := session.Extractor.Extract(line)
58 | for _, subdomain := range subdomains {
59 | results <- subscraping.Result{
60 | Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."),
61 | }
62 | s.results++
63 | }
64 | }
65 | }()
66 |
67 | return results
68 | }
69 |
70 | // Name returns the name of the source
71 | func (s *Source) Name() string {
72 | return "digitorus"
73 | }
74 |
75 | func (s *Source) IsDefault() bool {
76 | return true
77 | }
78 |
79 | func (s *Source) HasRecursiveSupport() bool {
80 | return true
81 | }
82 |
83 | func (s *Source) NeedsKey() bool {
84 | return false
85 | }
86 |
87 | func (s *Source) AddApiKeys(_ []string) {
88 | // no key needed
89 | }
90 |
91 | func (s *Source) Statistics() subscraping.Statistics {
92 | return subscraping.Statistics{
93 | Errors: s.errors,
94 | Results: s.results,
95 | TimeTaken: s.timeTaken,
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go:
--------------------------------------------------------------------------------
1 | // Package dnsdumpster logic
2 | package dnsdumpster
3 |
4 | import (
5 | "context"
6 | "encoding/json"
7 | "fmt"
8 | "time"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | type response struct {
14 | A []struct {
15 | Host string `json:"host"`
16 | } `json:"a"`
17 | Ns []struct {
18 | Host string `json:"host"`
19 | } `json:"ns"`
20 | }
21 |
22 | // Source is the passive scraping agent
23 | type Source struct {
24 | apiKeys []string
25 | timeTaken time.Duration
26 | errors int
27 | results int
28 | skipped bool
29 | }
30 |
31 | // Run function returns all subdomains found with the service
32 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
33 | results := make(chan subscraping.Result)
34 | s.errors = 0
35 | s.results = 0
36 |
37 | go func() {
38 | defer func(startTime time.Time) {
39 | s.timeTaken = time.Since(startTime)
40 | close(results)
41 | }(time.Now())
42 |
43 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
44 | if randomApiKey == "" {
45 | s.skipped = true
46 | return
47 | }
48 |
49 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey})
50 | if err != nil {
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
52 | s.errors++
53 | session.DiscardHTTPResponse(resp)
54 | return
55 | }
56 | defer session.DiscardHTTPResponse(resp)
57 |
58 | var response response
59 | err = json.NewDecoder(resp.Body).Decode(&response)
60 | if err != nil {
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
62 | s.errors++
63 | session.DiscardHTTPResponse(resp)
64 | return
65 | }
66 |
67 | for _, record := range append(response.A, response.Ns...) {
68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}
69 | s.results++
70 | }
71 |
72 | }()
73 |
74 | return results
75 | }
76 |
77 | // Name returns the name of the source
78 | func (s *Source) Name() string {
79 | return "dnsdumpster"
80 | }
81 |
82 | func (s *Source) IsDefault() bool {
83 | return true
84 | }
85 |
86 | func (s *Source) HasRecursiveSupport() bool {
87 | return false
88 | }
89 |
90 | func (s *Source) NeedsKey() bool {
91 | return true
92 | }
93 |
94 | func (s *Source) AddApiKeys(keys []string) {
95 | s.apiKeys = keys
96 | }
97 |
98 | func (s *Source) Statistics() subscraping.Statistics {
99 | return subscraping.Statistics{
100 | Errors: s.errors,
101 | Results: s.results,
102 | TimeTaken: s.timeTaken,
103 | Skipped: s.skipped,
104 | }
105 | }
106 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go:
--------------------------------------------------------------------------------
1 | package dnsrepo
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "io"
8 | "strings"
9 | "time"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | // Source is the passive scraping agent
15 | type Source struct {
16 | apiKeys []string
17 | timeTaken time.Duration
18 | errors int
19 | results int
20 | skipped bool
21 | }
22 |
23 | type DnsRepoResponse []struct {
24 | Domain string `json:"domain"`
25 | }
26 |
27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
28 | results := make(chan subscraping.Result)
29 | s.errors = 0
30 | s.results = 0
31 |
32 | go func() {
33 | defer func(startTime time.Time) {
34 | s.timeTaken = time.Since(startTime)
35 | close(results)
36 | }(time.Now())
37 |
38 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
39 | if randomApiKey == "" {
40 | s.skipped = true
41 | return
42 | }
43 |
44 | randomApiInfo := strings.Split(randomApiKey, ":")
45 | if len(randomApiInfo) != 2 {
46 | s.skipped = true
47 | return
48 | }
49 |
50 | token := randomApiInfo[0]
51 | apiKey := randomApiInfo[1]
52 |
53 | resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token})
54 | if err != nil {
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
56 | s.errors++
57 | session.DiscardHTTPResponse(resp)
58 | return
59 | }
60 | responseData, err := io.ReadAll(resp.Body)
61 | if err != nil {
62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
63 | s.errors++
64 | session.DiscardHTTPResponse(resp)
65 | return
66 | }
67 | session.DiscardHTTPResponse(resp)
68 | var result DnsRepoResponse
69 | err = json.Unmarshal(responseData, &result)
70 | if err != nil {
71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
72 | s.errors++
73 | session.DiscardHTTPResponse(resp)
74 | return
75 | }
76 | for _, sub := range result {
77 | results <- subscraping.Result{
78 | Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, "."),
79 | }
80 | s.results++
81 | }
82 |
83 | }()
84 |
85 | return results
86 | }
87 |
88 | // Name returns the name of the source
89 | func (s *Source) Name() string {
90 | return "dnsrepo"
91 | }
92 |
93 | func (s *Source) IsDefault() bool {
94 | return true
95 | }
96 |
97 | func (s *Source) HasRecursiveSupport() bool {
98 | return false
99 | }
100 |
101 | func (s *Source) NeedsKey() bool {
102 | return true
103 | }
104 |
105 | func (s *Source) AddApiKeys(keys []string) {
106 | s.apiKeys = keys
107 | }
108 |
109 | func (s *Source) Statistics() subscraping.Statistics {
110 | return subscraping.Statistics{
111 | Errors: s.errors,
112 | Results: s.results,
113 | TimeTaken: s.timeTaken,
114 | Skipped: s.skipped,
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/facebook/ctlogs_test.go:
--------------------------------------------------------------------------------
1 | package facebook
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "io"
7 | "os"
8 | "strings"
9 | "testing"
10 |
11 | "github.com/projectdiscovery/gologger"
12 | "github.com/projectdiscovery/retryablehttp-go"
13 | "github.com/projectdiscovery/utils/generic"
14 | )
15 |
16 | var (
17 | fb_API_ID = "$FB_APP_ID"
18 | fb_API_SECRET = "$FB_APP_SECRET"
19 | )
20 |
21 | func TestFacebookSource(t *testing.T) {
22 | if testing.Short() {
23 | t.Skip("skipping test in short mode.")
24 | }
25 |
26 | updateWithEnv(&fb_API_ID)
27 | updateWithEnv(&fb_API_SECRET)
28 | if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) {
29 | t.SkipNow()
30 | }
31 | k := apiKey{
32 | AppID: fb_API_ID,
33 | Secret: fb_API_SECRET,
34 | }
35 | k.FetchAccessToken()
36 | if k.Error != nil {
37 | t.Fatal(k.Error)
38 | }
39 |
40 | fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken)
41 | resp, err := retryablehttp.Get(fetchURL)
42 | if err != nil {
43 | t.Fatal(err)
44 | }
45 | defer func() {
46 | if err := resp.Body.Close(); err != nil {
47 | gologger.Error().Msgf("error closing response body: %s", err)
48 | }
49 | }()
50 | bin, err := io.ReadAll(resp.Body)
51 | if err != nil {
52 | t.Fatal(err)
53 | }
54 | response := &response{}
55 | if err := json.Unmarshal(bin, response); err != nil {
56 | t.Fatal(err)
57 | }
58 | if len(response.Data) == 0 {
59 | t.Fatal("no data found")
60 | }
61 | }
62 |
63 | func updateWithEnv(key *string) {
64 | if key == nil {
65 | return
66 | }
67 | value := *key
68 | if strings.HasPrefix(value, "quot;) {
69 | *key = os.Getenv(value[1:])
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/facebook/types.go:
--------------------------------------------------------------------------------
1 | package facebook
2 |
3 | type authResponse struct {
4 | AccessToken string `json:"access_token"`
5 | }
6 |
7 | /*
8 | {
9 | "data": [
10 | {
11 | "domains": [
12 | "docs.hackerone.com"
13 | ],
14 | "id": "10056051421102939"
15 | },
16 | ...
17 | ],
18 | "paging": {
19 | "cursors": {
20 | "before": "MTAwNTYwNTE0MjExMDI5MzkZD",
21 | "after": "Njc0OTczNTA5NTA1MzUxNwZDZD"
22 | },
23 | "next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD"
24 | }
25 | }
26 | */
27 | // example response
28 |
29 | type response struct {
30 | Data []struct {
31 | Domains []string `json:"domains"`
32 | } `json:"data"`
33 | Paging struct {
34 | Next string `json:"next"`
35 | } `json:"paging"`
36 | }
37 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/fofa/fofa.go:
--------------------------------------------------------------------------------
1 | // Package fofa logic
2 | package fofa
3 |
4 | import (
5 | "context"
6 | "encoding/base64"
7 | "fmt"
8 | "regexp"
9 | "strings"
10 | "time"
11 |
12 | jsoniter "github.com/json-iterator/go"
13 |
14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
15 | )
16 |
17 | type fofaResponse struct {
18 | Error bool `json:"error"`
19 | ErrMsg string `json:"errmsg"`
20 | Size int `json:"size"`
21 | Results []string `json:"results"`
22 | }
23 |
24 | // Source is the passive scraping agent
25 | type Source struct {
26 | apiKeys []apiKey
27 | timeTaken time.Duration
28 | errors int
29 | results int
30 | skipped bool
31 | }
32 |
33 | type apiKey struct {
34 | username string
35 | secret string
36 | }
37 |
38 | // Run function returns all subdomains found with the service
39 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
40 | results := make(chan subscraping.Result)
41 | s.errors = 0
42 | s.results = 0
43 |
44 | go func() {
45 | defer func(startTime time.Time) {
46 | s.timeTaken = time.Since(startTime)
47 | close(results)
48 | }(time.Now())
49 |
50 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
51 | if randomApiKey.username == "" || randomApiKey.secret == "" {
52 | s.skipped = true
53 | return
54 | }
55 |
56 | // fofa api doc https://fofa.info/static_pages/api_help
57 | qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain))
58 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64))
59 | if err != nil && resp == nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 | s.errors++
62 | session.DiscardHTTPResponse(resp)
63 | return
64 | }
65 |
66 | var response fofaResponse
67 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
68 | if err != nil {
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
70 | s.errors++
71 | session.DiscardHTTPResponse(resp)
72 | return
73 | }
74 | session.DiscardHTTPResponse(resp)
75 |
76 | if response.Error {
77 | results <- subscraping.Result{
78 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg),
79 | }
80 | s.errors++
81 | return
82 | }
83 |
84 | if response.Size > 0 {
85 | for _, subdomain := range response.Results {
86 | if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") {
87 | subdomain = subdomain[strings.Index(subdomain, "//")+2:]
88 | }
89 | re := regexp.MustCompile(`:\d+
)
90 | if re.MatchString(subdomain) {
91 | subdomain = re.ReplaceAllString(subdomain, "")
92 | }
93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
94 | s.results++
95 | }
96 | }
97 | }()
98 |
99 | return results
100 | }
101 |
102 | // Name returns the name of the source
103 | func (s *Source) Name() string {
104 | return "fofa"
105 | }
106 |
107 | func (s *Source) IsDefault() bool {
108 | return true
109 | }
110 |
111 | func (s *Source) HasRecursiveSupport() bool {
112 | return false
113 | }
114 |
115 | func (s *Source) NeedsKey() bool {
116 | return true
117 | }
118 |
119 | func (s *Source) AddApiKeys(keys []string) {
120 | s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
121 | return apiKey{k, v}
122 | })
123 | }
124 |
125 | func (s *Source) Statistics() subscraping.Statistics {
126 | return subscraping.Statistics{
127 | Errors: s.errors,
128 | Results: s.results,
129 | TimeTaken: s.timeTaken,
130 | Skipped: s.skipped,
131 | }
132 | }
133 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/fullhunt/fullhunt.go:
--------------------------------------------------------------------------------
1 | package fullhunt
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "time"
7 |
8 | jsoniter "github.com/json-iterator/go"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | // fullhunt response
14 | type fullHuntResponse struct {
15 | Hosts []string `json:"hosts"`
16 | Message string `json:"message"`
17 | Status int `json:"status"`
18 | }
19 |
20 | // Source is the passive scraping agent
21 | type Source struct {
22 | apiKeys []string
23 | timeTaken time.Duration
24 | errors int
25 | results int
26 | skipped bool
27 | }
28 |
29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
30 | results := make(chan subscraping.Result)
31 | s.errors = 0
32 | s.results = 0
33 |
34 | go func() {
35 | defer func(startTime time.Time) {
36 | s.timeTaken = time.Since(startTime)
37 | close(results)
38 | }(time.Now())
39 |
40 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
41 | if randomApiKey == "" {
42 | s.skipped = true
43 | return
44 | }
45 |
46 | resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey})
47 | if err != nil {
48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
49 | s.errors++
50 | session.DiscardHTTPResponse(resp)
51 | return
52 | }
53 |
54 | var response fullHuntResponse
55 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
56 | if err != nil {
57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
58 | s.errors++
59 | session.DiscardHTTPResponse(resp)
60 | return
61 | }
62 | session.DiscardHTTPResponse(resp)
63 | for _, record := range response.Hosts {
64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}
65 | s.results++
66 | }
67 | }()
68 |
69 | return results
70 | }
71 |
72 | // Name returns the name of the source
73 | func (s *Source) Name() string {
74 | return "fullhunt"
75 | }
76 |
77 | func (s *Source) IsDefault() bool {
78 | return true
79 | }
80 |
81 | func (s *Source) HasRecursiveSupport() bool {
82 | return false
83 | }
84 |
85 | func (s *Source) NeedsKey() bool {
86 | return true
87 | }
88 |
89 | func (s *Source) AddApiKeys(keys []string) {
90 | s.apiKeys = keys
91 | }
92 |
93 | func (s *Source) Statistics() subscraping.Statistics {
94 | return subscraping.Statistics{
95 | Errors: s.errors,
96 | Results: s.results,
97 | TimeTaken: s.timeTaken,
98 | Skipped: s.skipped,
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/github/tokenmanager.go:
--------------------------------------------------------------------------------
1 | package github
2 |
3 | import "time"
4 |
5 | // Token struct
6 | type Token struct {
7 | Hash string
8 | RetryAfter int64
9 | ExceededTime time.Time
10 | }
11 |
12 | // Tokens is the internal struct to manage the current token
13 | // and the pool
14 | type Tokens struct {
15 | current int
16 | pool []Token
17 | }
18 |
19 | // NewTokenManager initialize the tokens pool
20 | func NewTokenManager(keys []string) *Tokens {
21 | pool := []Token{}
22 | for _, key := range keys {
23 | t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0}
24 | pool = append(pool, t)
25 | }
26 |
27 | return &Tokens{
28 | current: 0,
29 | pool: pool,
30 | }
31 | }
32 |
33 | func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) {
34 | if r.current >= len(r.pool) {
35 | r.current %= len(r.pool)
36 | }
37 | if r.pool[r.current].RetryAfter == 0 {
38 | r.pool[r.current].ExceededTime = time.Now()
39 | r.pool[r.current].RetryAfter = retryAfter
40 | }
41 | }
42 |
43 | // Get returns a new token from the token pool
44 | func (r *Tokens) Get() *Token {
45 | resetExceededTokens(r)
46 |
47 | if r.current >= len(r.pool) {
48 | r.current %= len(r.pool)
49 | }
50 |
51 | result := &r.pool[r.current]
52 | r.current++
53 |
54 | return result
55 | }
56 |
57 | func resetExceededTokens(r *Tokens) {
58 | for i, token := range r.pool {
59 | if token.RetryAfter > 0 {
60 | if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter {
61 | r.pool[i].ExceededTime = time.Time{}
62 | r.pool[i].RetryAfter = 0
63 | }
64 | }
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/gitlab/gitlab.go:
--------------------------------------------------------------------------------
1 | package gitlab
2 |
3 | import (
4 | "bufio"
5 | "context"
6 | "fmt"
7 | "net/http"
8 | "net/url"
9 | "regexp"
10 | "strings"
11 | "sync"
12 | "time"
13 |
14 | jsoniter "github.com/json-iterator/go"
15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
16 | "github.com/tomnomnom/linkheader"
17 | )
18 |
19 | // Source is the passive scraping agent
20 | type Source struct {
21 | apiKeys []string
22 | timeTaken time.Duration
23 | errors int
24 | results int
25 | skipped bool
26 | }
27 |
28 | type item struct {
29 | Data string `json:"data"`
30 | ProjectId int `json:"project_id"`
31 | Path string `json:"path"`
32 | Ref string `json:"ref"`
33 | }
34 |
35 | // Run function returns all subdomains found with the service
36 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
37 | results := make(chan subscraping.Result)
38 | s.errors = 0
39 | s.results = 0
40 |
41 | go func() {
42 | defer func(startTime time.Time) {
43 | s.timeTaken = time.Since(startTime)
44 | close(results)
45 | }(time.Now())
46 |
47 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
48 | if randomApiKey == "" {
49 | return
50 | }
51 |
52 | headers := map[string]string{"PRIVATE-TOKEN": randomApiKey}
53 |
54 | searchURL := fmt.Sprintf("https://gitlab.com/api/v4/search?scope=blobs&search=%s&per_page=100", domain)
55 | s.enumerate(ctx, searchURL, domainRegexp(domain), headers, session, results)
56 |
57 | }()
58 |
59 | return results
60 | }
61 |
62 | func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) {
63 | select {
64 | case <-ctx.Done():
65 | return
66 | default:
67 | }
68 |
69 | resp, err := session.Get(ctx, searchURL, "", headers)
70 | if err != nil && resp == nil {
71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
72 | s.errors++
73 | session.DiscardHTTPResponse(resp)
74 | return
75 | }
76 |
77 | defer session.DiscardHTTPResponse(resp)
78 |
79 | var items []item
80 | err = jsoniter.NewDecoder(resp.Body).Decode(&items)
81 | if err != nil {
82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
83 | s.errors++
84 | return
85 | }
86 |
87 | var wg sync.WaitGroup
88 | wg.Add(len(items))
89 |
90 | for _, it := range items {
91 | go func(item item) {
92 | // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path
93 | fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref)
94 | resp, err := session.Get(ctx, fileUrl, "", headers)
95 | if err != nil {
96 | if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) {
97 | session.DiscardHTTPResponse(resp)
98 |
99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
100 | s.errors++
101 | return
102 | }
103 | }
104 |
105 | if resp.StatusCode == http.StatusOK {
106 | scanner := bufio.NewScanner(resp.Body)
107 | for scanner.Scan() {
108 | line := scanner.Text()
109 | if line == "" {
110 | continue
111 | }
112 | for _, subdomain := range domainRegexp.FindAllString(line, -1) {
113 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
114 | s.results++
115 | }
116 | }
117 | session.DiscardHTTPResponse(resp)
118 | }
119 | defer wg.Done()
120 | }(it)
121 | }
122 |
123 | // Links header, first, next, last...
124 | linksHeader := linkheader.Parse(resp.Header.Get("Link"))
125 | // Process the next link recursively
126 | for _, link := range linksHeader {
127 | if link.Rel == "next" {
128 | nextURL, err := url.QueryUnescape(link.URL)
129 | if err != nil {
130 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
131 | s.errors++
132 | return
133 | }
134 |
135 | s.enumerate(ctx, nextURL, domainRegexp, headers, session, results)
136 | }
137 | }
138 |
139 | wg.Wait()
140 | }
141 |
142 | func domainRegexp(domain string) *regexp.Regexp {
143 | rdomain := strings.ReplaceAll(domain, ".", "\\.")
144 | return regexp.MustCompile("(\\w[a-zA-Z0-9][a-zA-Z0-9-\\.]*)" + rdomain)
145 | }
146 |
147 | // Name returns the name of the source
148 | func (s *Source) Name() string {
149 | return "gitlab"
150 | }
151 |
152 | func (s *Source) IsDefault() bool {
153 | return false
154 | }
155 |
156 | func (s *Source) HasRecursiveSupport() bool {
157 | return false
158 | }
159 |
160 | func (s *Source) NeedsKey() bool {
161 | return true
162 | }
163 |
164 | func (s *Source) AddApiKeys(keys []string) {
165 | s.apiKeys = keys
166 | }
167 |
168 | // Statistics returns the statistics for the source
169 | func (s *Source) Statistics() subscraping.Statistics {
170 | return subscraping.Statistics{
171 | Errors: s.errors,
172 | Results: s.results,
173 | TimeTaken: s.timeTaken,
174 | Skipped: s.skipped,
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/hackertarget/hackertarget.go:
--------------------------------------------------------------------------------
1 | // Package hackertarget logic
2 | package hackertarget
3 |
4 | import (
5 | "bufio"
6 | "context"
7 | "fmt"
8 | "time"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | // Source is the passive scraping agent
14 | type Source struct {
15 | timeTaken time.Duration
16 | errors int
17 | results int
18 | }
19 |
20 | // Run function returns all subdomains found with the service
21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
22 | results := make(chan subscraping.Result)
23 | s.errors = 0
24 | s.results = 0
25 |
26 | go func() {
27 | defer func(startTime time.Time) {
28 | s.timeTaken = time.Since(startTime)
29 | close(results)
30 | }(time.Now())
31 |
32 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain))
33 | if err != nil {
34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
35 | s.errors++
36 | session.DiscardHTTPResponse(resp)
37 | return
38 | }
39 |
40 | defer func() {
41 | if err := resp.Body.Close(); err != nil {
42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
43 | s.errors++
44 | }
45 | }()
46 |
47 | scanner := bufio.NewScanner(resp.Body)
48 | for scanner.Scan() {
49 | line := scanner.Text()
50 | if line == "" {
51 | continue
52 | }
53 | match := session.Extractor.Extract(line)
54 | for _, subdomain := range match {
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
56 | s.results++
57 | }
58 | }
59 | }()
60 |
61 | return results
62 | }
63 |
64 | // Name returns the name of the source
65 | func (s *Source) Name() string {
66 | return "hackertarget"
67 | }
68 |
69 | func (s *Source) IsDefault() bool {
70 | return true
71 | }
72 |
73 | func (s *Source) HasRecursiveSupport() bool {
74 | return true
75 | }
76 |
77 | func (s *Source) NeedsKey() bool {
78 | return false
79 | }
80 |
81 | func (s *Source) AddApiKeys(_ []string) {
82 | // no key needed
83 | }
84 |
85 | func (s *Source) Statistics() subscraping.Statistics {
86 | return subscraping.Statistics{
87 | Errors: s.errors,
88 | Results: s.results,
89 | TimeTaken: s.timeTaken,
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go:
--------------------------------------------------------------------------------
1 | // Package hudsonrock logic
2 | package hudsonrock
3 |
4 | import (
5 | "context"
6 | "encoding/json"
7 | "fmt"
8 | "time"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | type hudsonrockResponse struct {
14 | Data struct {
15 | EmployeesUrls []struct {
16 | URL string `json:"url"`
17 | } `json:"employees_urls"`
18 | ClientsUrls []struct {
19 | URL string `json:"url"`
20 | } `json:"clients_urls"`
21 | } `json:"data"`
22 | }
23 |
24 | // Source is the passive scraping agent
25 | type Source struct {
26 | timeTaken time.Duration
27 | errors int
28 | results int
29 | }
30 |
31 | // Run function returns all subdomains found with the service
32 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
33 | results := make(chan subscraping.Result)
34 | s.errors = 0
35 | s.results = 0
36 |
37 | go func() {
38 | defer func(startTime time.Time) {
39 | s.timeTaken = time.Since(startTime)
40 | close(results)
41 | }(time.Now())
42 |
43 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain))
44 | if err != nil {
45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
46 | s.errors++
47 | session.DiscardHTTPResponse(resp)
48 | return
49 | }
50 | defer session.DiscardHTTPResponse(resp)
51 |
52 | var response hudsonrockResponse
53 | err = json.NewDecoder(resp.Body).Decode(&response)
54 | if err != nil {
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
56 | s.errors++
57 | session.DiscardHTTPResponse(resp)
58 | return
59 | }
60 |
61 | for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) {
62 | for _, subdomain := range session.Extractor.Extract(record.URL) {
63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
64 | s.results++
65 | }
66 | }
67 |
68 | }()
69 |
70 | return results
71 | }
72 |
73 | // Name returns the name of the source
74 | func (s *Source) Name() string {
75 | return "hudsonrock"
76 | }
77 |
78 | func (s *Source) IsDefault() bool {
79 | return false
80 | }
81 |
82 | func (s *Source) HasRecursiveSupport() bool {
83 | return false
84 | }
85 |
86 | func (s *Source) NeedsKey() bool {
87 | return false
88 | }
89 |
90 | func (s *Source) AddApiKeys(_ []string) {
91 | // no key needed
92 | }
93 |
94 | func (s *Source) Statistics() subscraping.Statistics {
95 | return subscraping.Statistics{
96 | Errors: s.errors,
97 | Results: s.results,
98 | TimeTaken: s.timeTaken,
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/hunter/hunter.go:
--------------------------------------------------------------------------------
1 | package hunter
2 |
3 | import (
4 | "context"
5 | "encoding/base64"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | type hunterResp struct {
14 | Code int `json:"code"`
15 | Data hunterData `json:"data"`
16 | Message string `json:"message"`
17 | }
18 |
19 | type infoArr struct {
20 | URL string `json:"url"`
21 | IP string `json:"ip"`
22 | Port int `json:"port"`
23 | Domain string `json:"domain"`
24 | Protocol string `json:"protocol"`
25 | }
26 |
27 | type hunterData struct {
28 | InfoArr []infoArr `json:"arr"`
29 | Total int `json:"total"`
30 | }
31 |
32 | // Source is the passive scraping agent
33 | type Source struct {
34 | apiKeys []string
35 | timeTaken time.Duration
36 | errors int
37 | results int
38 | skipped bool
39 | }
40 |
41 | // Run function returns all subdomains found with the service
42 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
43 | results := make(chan subscraping.Result)
44 | s.errors = 0
45 | s.results = 0
46 |
47 | go func() {
48 | defer func(startTime time.Time) {
49 | s.timeTaken = time.Since(startTime)
50 | close(results)
51 | }(time.Now())
52 |
53 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
54 | if randomApiKey == "" {
55 | s.skipped = true
56 | return
57 | }
58 |
59 | var pages = 1
60 | for currentPage := 1; currentPage <= pages; currentPage++ {
61 | // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2
62 | qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain))
63 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage))
64 | if err != nil && resp == nil {
65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
66 | s.errors++
67 | session.DiscardHTTPResponse(resp)
68 | return
69 | }
70 |
71 | var response hunterResp
72 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
73 | if err != nil {
74 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
75 | s.errors++
76 | session.DiscardHTTPResponse(resp)
77 | return
78 | }
79 | session.DiscardHTTPResponse(resp)
80 |
81 | if response.Code == 401 || response.Code == 400 {
82 | results <- subscraping.Result{
83 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
84 | }
85 | s.errors++
86 | return
87 | }
88 |
89 | if response.Data.Total > 0 {
90 | for _, hunterInfo := range response.Data.InfoArr {
91 | subdomain := hunterInfo.Domain
92 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
93 | s.results++
94 | }
95 | }
96 | pages = int(response.Data.Total/1000) + 1
97 | }
98 | }()
99 |
100 | return results
101 | }
102 |
103 | // Name returns the name of the source
104 | func (s *Source) Name() string {
105 | return "hunter"
106 | }
107 |
108 | func (s *Source) IsDefault() bool {
109 | return true
110 | }
111 |
112 | func (s *Source) HasRecursiveSupport() bool {
113 | return false
114 | }
115 |
116 | func (s *Source) NeedsKey() bool {
117 | return true
118 | }
119 |
120 | func (s *Source) AddApiKeys(keys []string) {
121 | s.apiKeys = keys
122 | }
123 |
124 | func (s *Source) Statistics() subscraping.Statistics {
125 | return subscraping.Statistics{
126 | Errors: s.errors,
127 | Results: s.results,
128 | TimeTaken: s.timeTaken,
129 | Skipped: s.skipped,
130 | }
131 | }
132 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/intelx/intelx.go:
--------------------------------------------------------------------------------
1 | // Package intelx logic
2 | package intelx
3 |
4 | import (
5 | "bytes"
6 | "context"
7 | "encoding/json"
8 | "fmt"
9 | "io"
10 | "time"
11 |
12 | jsoniter "github.com/json-iterator/go"
13 |
14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
15 | )
16 |
17 | type searchResponseType struct {
18 | ID string `json:"id"`
19 | Status int `json:"status"`
20 | }
21 |
22 | type selectorType struct {
23 | Selectvalue string `json:"selectorvalue"`
24 | }
25 |
26 | type searchResultType struct {
27 | Selectors []selectorType `json:"selectors"`
28 | Status int `json:"status"`
29 | }
30 |
31 | type requestBody struct {
32 | Term string
33 | Maxresults int
34 | Media int
35 | Target int
36 | Terminate []int
37 | Timeout int
38 | }
39 |
40 | // Source is the passive scraping agent
41 | type Source struct {
42 | apiKeys []apiKey
43 | timeTaken time.Duration
44 | errors int
45 | results int
46 | skipped bool
47 | }
48 |
49 | type apiKey struct {
50 | host string
51 | key string
52 | }
53 |
54 | // Run function returns all subdomains found with the service
55 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
56 | results := make(chan subscraping.Result)
57 | s.errors = 0
58 | s.results = 0
59 |
60 | go func() {
61 | defer func(startTime time.Time) {
62 | s.timeTaken = time.Since(startTime)
63 | close(results)
64 | }(time.Now())
65 |
66 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
67 | if randomApiKey.host == "" || randomApiKey.key == "" {
68 | s.skipped = true
69 | return
70 | }
71 |
72 | searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", randomApiKey.host, randomApiKey.key)
73 | reqBody := requestBody{
74 | Term: domain,
75 | Maxresults: 100000,
76 | Media: 0,
77 | Target: 1,
78 | Timeout: 20,
79 | }
80 |
81 | body, err := json.Marshal(reqBody)
82 | if err != nil {
83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
84 | s.errors++
85 | return
86 | }
87 |
88 | resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body))
89 | if err != nil {
90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
91 | s.errors++
92 | session.DiscardHTTPResponse(resp)
93 | return
94 | }
95 |
96 | var response searchResponseType
97 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
98 | if err != nil {
99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
100 | s.errors++
101 | session.DiscardHTTPResponse(resp)
102 | return
103 | }
104 |
105 | session.DiscardHTTPResponse(resp)
106 |
107 | resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID)
108 | status := 0
109 | for status == 0 || status == 3 {
110 | resp, err = session.Get(ctx, resultsURL, "", nil)
111 | if err != nil {
112 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
113 | s.errors++
114 | session.DiscardHTTPResponse(resp)
115 | return
116 | }
117 | var response searchResultType
118 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
119 | if err != nil {
120 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
121 | s.errors++
122 | session.DiscardHTTPResponse(resp)
123 | return
124 | }
125 |
126 | _, err = io.ReadAll(resp.Body)
127 | if err != nil {
128 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
129 | s.errors++
130 | session.DiscardHTTPResponse(resp)
131 | return
132 | }
133 | session.DiscardHTTPResponse(resp)
134 |
135 | status = response.Status
136 | for _, hostname := range response.Selectors {
137 | results <- subscraping.Result{
138 | Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue,
139 | }
140 | s.results++
141 | }
142 | }
143 | }()
144 |
145 | return results
146 | }
147 |
148 | // Name returns the name of the source
149 | func (s *Source) Name() string {
150 | return "intelx"
151 | }
152 |
153 | func (s *Source) IsDefault() bool {
154 | return true
155 | }
156 |
157 | func (s *Source) HasRecursiveSupport() bool {
158 | return false
159 | }
160 |
161 | func (s *Source) NeedsKey() bool {
162 | return true
163 | }
164 |
165 | func (s *Source) AddApiKeys(keys []string) {
166 | s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
167 | return apiKey{k, v}
168 | })
169 | }
170 |
171 | func (s *Source) Statistics() subscraping.Statistics {
172 | return subscraping.Statistics{
173 | Errors: s.errors,
174 | Results: s.results,
175 | TimeTaken: s.timeTaken,
176 | Skipped: s.skipped,
177 | }
178 | }
179 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/leakix/leakix.go:
--------------------------------------------------------------------------------
1 | // Package leakix logic
2 | package leakix
3 |
4 | import (
5 | "context"
6 | "encoding/json"
7 | "fmt"
8 | "time"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | // Source is the passive scraping agent
14 | type Source struct {
15 | apiKeys []string
16 | timeTaken time.Duration
17 | errors int
18 | results int
19 | skipped bool
20 | }
21 |
22 | // Run function returns all subdomains found with the service
23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
24 | results := make(chan subscraping.Result)
25 | s.errors = 0
26 | s.results = 0
27 |
28 | go func() {
29 | defer func(startTime time.Time) {
30 | s.timeTaken = time.Since(startTime)
31 | close(results)
32 | }(time.Now())
33 | // Default headers
34 | headers := map[string]string{
35 | "accept": "application/json",
36 | }
37 | // Pick an API key
38 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
39 | if randomApiKey != "" {
40 | headers["api-key"] = randomApiKey
41 | }
42 | // Request
43 | resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers)
44 | if err != nil {
45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
46 | s.errors++
47 | return
48 | }
49 |
50 | defer session.DiscardHTTPResponse(resp)
51 |
52 | if resp.StatusCode != 200 {
53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)}
54 | s.errors++
55 | return
56 | }
57 | // Parse and return results
58 | var subdomains []subResponse
59 | decoder := json.NewDecoder(resp.Body)
60 | err = decoder.Decode(&subdomains)
61 | if err != nil {
62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
63 | s.errors++
64 | return
65 | }
66 | for _, result := range subdomains {
67 | results <- subscraping.Result{
68 | Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain,
69 | }
70 | s.results++
71 | }
72 | }()
73 | return results
74 | }
75 |
76 | // Name returns the name of the source
77 | func (s *Source) Name() string {
78 | return "leakix"
79 | }
80 |
81 | func (s *Source) IsDefault() bool {
82 | return true
83 | }
84 |
85 | func (s *Source) HasRecursiveSupport() bool {
86 | return true
87 | }
88 |
89 | func (s *Source) NeedsKey() bool {
90 | return true
91 | }
92 |
93 | func (s *Source) AddApiKeys(keys []string) {
94 | s.apiKeys = keys
95 | }
96 |
97 | func (s *Source) Statistics() subscraping.Statistics {
98 | return subscraping.Statistics{
99 | Errors: s.errors,
100 | Results: s.results,
101 | TimeTaken: s.timeTaken,
102 | Skipped: s.skipped,
103 | }
104 | }
105 |
106 | type subResponse struct {
107 | Subdomain string `json:"subdomain"`
108 | DistinctIps int `json:"distinct_ips"`
109 | LastSeen time.Time `json:"last_seen"`
110 | }
111 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/pugrecon/pugrecon.go:
--------------------------------------------------------------------------------
1 | // Package pugrecon logic
2 | package pugrecon
3 |
4 | import (
5 | "bytes"
6 | "context"
7 | "encoding/json"
8 | "fmt"
9 | "net/http"
10 | "time"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | // pugreconResult stores a single result from the pugrecon API
16 | type pugreconResult struct {
17 | Name string `json:"name"`
18 | }
19 |
20 | // pugreconAPIResponse stores the response from the pugrecon API
21 | type pugreconAPIResponse struct {
22 | Results []pugreconResult `json:"results"`
23 | QuotaRemaining int `json:"quota_remaining"`
24 | Limited bool `json:"limited"`
25 | TotalResults int `json:"total_results"`
26 | Message string `json:"message"`
27 | }
28 |
29 | // Source is the passive scraping agent
30 | type Source struct {
31 | apiKeys []string
32 | timeTaken time.Duration
33 | errors int
34 | results int
35 | skipped bool
36 | }
37 |
38 | // Run function returns all subdomains found with the service
39 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
40 | results := make(chan subscraping.Result)
41 | s.errors = 0
42 | s.results = 0
43 |
44 | go func() {
45 | defer func(startTime time.Time) {
46 | s.timeTaken = time.Since(startTime)
47 | close(results)
48 | }(time.Now())
49 |
50 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
51 | if randomApiKey == "" {
52 | s.skipped = true
53 | return
54 | }
55 |
56 | // Prepare POST request data
57 | postData := map[string]string{"domain_name": domain}
58 | bodyBytes, err := json.Marshal(postData)
59 | if err != nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)}
61 | s.errors++
62 | return
63 | }
64 | bodyReader := bytes.NewReader(bodyBytes)
65 |
66 | // Prepare headers
67 | headers := map[string]string{
68 | "Authorization": "Bearer " + randomApiKey,
69 | "Content-Type": "application/json",
70 | "Accept": "application/json",
71 | }
72 |
73 | apiURL := "https://pugrecon.com/api/v1/domains"
74 | resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control
75 | if err != nil {
76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
77 | s.errors++
78 | session.DiscardHTTPResponse(resp)
79 | return
80 | }
81 | defer func() {
82 | if err := resp.Body.Close(); err != nil {
83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to close response body: %w", err)}
84 | s.errors++
85 | }
86 | }()
87 |
88 | if resp.StatusCode != http.StatusOK {
89 | errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode)
90 | // Attempt to read error message from body if possible
91 | var apiResp pugreconAPIResponse
92 | if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" {
93 | errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message)
94 | }
95 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", errorMsg)}
96 | s.errors++
97 | return
98 | }
99 |
100 | var response pugreconAPIResponse
101 | err = json.NewDecoder(resp.Body).Decode(&response)
102 | if err != nil {
103 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
104 | s.errors++
105 | return
106 | }
107 |
108 | for _, subdomain := range response.Results {
109 | results <- subscraping.Result{
110 | Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name,
111 | }
112 | s.results++
113 | }
114 | }()
115 |
116 | return results
117 | }
118 |
119 | // Name returns the name of the source
120 | func (s *Source) Name() string {
121 | return "pugrecon"
122 | }
123 |
124 | // IsDefault returns false as this is not a default source.
125 | func (s *Source) IsDefault() bool {
126 | return false
127 | }
128 |
129 | // HasRecursiveSupport returns false as this source does not support recursive searches.
130 | func (s *Source) HasRecursiveSupport() bool {
131 | return false
132 | }
133 |
134 | // NeedsKey returns true as this source requires an API key.
135 | func (s *Source) NeedsKey() bool {
136 | return true
137 | }
138 |
139 | // AddApiKeys adds the API keys for the source.
140 | func (s *Source) AddApiKeys(keys []string) {
141 | s.apiKeys = keys
142 | }
143 |
144 | // Statistics returns the statistics for the source.
145 | func (s *Source) Statistics() subscraping.Statistics {
146 | return subscraping.Statistics{
147 | Errors: s.errors,
148 | Results: s.results,
149 | TimeTaken: s.timeTaken,
150 | Skipped: s.skipped,
151 | }
152 | }
153 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/quake/quake.go:
--------------------------------------------------------------------------------
1 | // Package quake logic
2 | package quake
3 |
4 | import (
5 | "bytes"
6 | "context"
7 | "fmt"
8 | "strings"
9 | "time"
10 |
11 | jsoniter "github.com/json-iterator/go"
12 |
13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
14 | )
15 |
16 | type quakeResults struct {
17 | Code int `json:"code"`
18 | Message string `json:"message"`
19 | Data []struct {
20 | Service struct {
21 | HTTP struct {
22 | Host string `json:"host"`
23 | } `json:"http"`
24 | }
25 | } `json:"data"`
26 | Meta struct {
27 | Pagination struct {
28 | Total int `json:"total"`
29 | } `json:"pagination"`
30 | } `json:"meta"`
31 | }
32 |
33 | // Source is the passive scraping agent
34 | type Source struct {
35 | apiKeys []string
36 | timeTaken time.Duration
37 | errors int
38 | results int
39 | skipped bool
40 | }
41 |
42 | // Run function returns all subdomains found with the service
43 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
44 | results := make(chan subscraping.Result)
45 | s.errors = 0
46 | s.results = 0
47 |
48 | go func() {
49 | defer func(startTime time.Time) {
50 | s.timeTaken = time.Since(startTime)
51 | close(results)
52 | }(time.Now())
53 |
54 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
55 | if randomApiKey == "" {
56 | s.skipped = true
57 | return
58 | }
59 |
60 | // quake api doc https://quake.360.cn/quake/#/help
61 | var pageSize = 500
62 | var start = 0
63 | var totalResults = -1
64 |
65 | for {
66 | var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start)
67 | resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{
68 | "Content-Type": "application/json", "X-QuakeToken": randomApiKey,
69 | }, bytes.NewReader(requestBody))
70 | if err != nil {
71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
72 | s.errors++
73 | session.DiscardHTTPResponse(resp)
74 | return
75 | }
76 |
77 | var response quakeResults
78 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
79 | if err != nil {
80 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
81 | s.errors++
82 | session.DiscardHTTPResponse(resp)
83 | return
84 | }
85 | session.DiscardHTTPResponse(resp)
86 |
87 | if response.Code != 0 {
88 | results <- subscraping.Result{
89 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
90 | }
91 | s.errors++
92 | return
93 | }
94 |
95 | if totalResults == -1 {
96 | totalResults = response.Meta.Pagination.Total
97 | }
98 |
99 | for _, quakeDomain := range response.Data {
100 | subdomain := quakeDomain.Service.HTTP.Host
101 | if strings.ContainsAny(subdomain, "暂无权限") {
102 | continue
103 | }
104 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
105 | s.results++
106 | }
107 |
108 | if len(response.Data) == 0 || start+pageSize >= totalResults {
109 | break
110 | }
111 |
112 | start += pageSize
113 | }
114 | }()
115 |
116 | return results
117 | }
118 |
119 | // Name returns the name of the source
120 | func (s *Source) Name() string {
121 | return "quake"
122 | }
123 |
124 | func (s *Source) IsDefault() bool {
125 | return true
126 | }
127 |
128 | func (s *Source) HasRecursiveSupport() bool {
129 | return false
130 | }
131 |
132 | func (s *Source) NeedsKey() bool {
133 | return true
134 | }
135 |
136 | func (s *Source) AddApiKeys(keys []string) {
137 | s.apiKeys = keys
138 | }
139 |
140 | func (s *Source) Statistics() subscraping.Statistics {
141 | return subscraping.Statistics{
142 | Errors: s.errors,
143 | Results: s.results,
144 | TimeTaken: s.timeTaken,
145 | Skipped: s.skipped,
146 | }
147 | }
148 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/rapiddns/rapiddns.go:
--------------------------------------------------------------------------------
1 | // Package rapiddns is a RapidDNS Scraping Engine in Golang
2 | package rapiddns
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "io"
8 | "regexp"
9 | "strconv"
10 | "time"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | var pagePattern = regexp.MustCompile(`class="page-link" href="/subdomain/[^"]+\?page=(\d+)">`)
16 |
17 | // Source is the passive scraping agent
18 | type Source struct {
19 | timeTaken time.Duration
20 | errors int
21 | results int
22 | }
23 |
24 | // Run function returns all subdomains found with the service
25 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
26 | results := make(chan subscraping.Result)
27 | s.errors = 0
28 | s.results = 0
29 |
30 | go func() {
31 | defer func(startTime time.Time) {
32 | s.timeTaken = time.Since(startTime)
33 | close(results)
34 | }(time.Now())
35 |
36 | page := 1
37 | maxPages := 1
38 | for {
39 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page))
40 | if err != nil {
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
42 | s.errors++
43 | session.DiscardHTTPResponse(resp)
44 | return
45 | }
46 |
47 | body, err := io.ReadAll(resp.Body)
48 | if err != nil {
49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
50 | s.errors++
51 | session.DiscardHTTPResponse(resp)
52 | return
53 | }
54 |
55 | session.DiscardHTTPResponse(resp)
56 |
57 | src := string(body)
58 | for _, subdomain := range session.Extractor.Extract(src) {
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
60 | s.results++
61 | }
62 |
63 | if maxPages == 1 {
64 | matches := pagePattern.FindAllStringSubmatch(src, -1)
65 | if len(matches) > 0 {
66 | lastMatch := matches[len(matches)-1]
67 | if len(lastMatch) > 1 {
68 | maxPages, _ = strconv.Atoi(lastMatch[1])
69 | }
70 | }
71 | }
72 |
73 | if page >= maxPages {
74 | break
75 | }
76 | page++
77 | }
78 | }()
79 |
80 | return results
81 | }
82 |
83 | // Name returns the name of the source
84 | func (s *Source) Name() string {
85 | return "rapiddns"
86 | }
87 |
88 | func (s *Source) IsDefault() bool {
89 | return false
90 | }
91 |
92 | func (s *Source) HasRecursiveSupport() bool {
93 | return false
94 | }
95 |
96 | func (s *Source) NeedsKey() bool {
97 | return false
98 | }
99 |
100 | func (s *Source) AddApiKeys(_ []string) {
101 | // no key needed
102 | }
103 |
104 | func (s *Source) Statistics() subscraping.Statistics {
105 | return subscraping.Statistics{
106 | Errors: s.errors,
107 | Results: s.results,
108 | TimeTaken: s.timeTaken,
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/reconcloud/reconcloud.go:
--------------------------------------------------------------------------------
1 | // Package reconcloud logic
2 | package reconcloud
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | type reconCloudResponse struct {
14 | MsgType string `json:"msg_type"`
15 | RequestID string `json:"request_id"`
16 | OnCache bool `json:"on_cache"`
17 | Step string `json:"step"`
18 | CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"`
19 | }
20 |
21 | type cloudAssetsList struct {
22 | Key string `json:"key"`
23 | Domain string `json:"domain"`
24 | CloudProvider string `json:"cloud_provider"`
25 | }
26 |
27 | // Source is the passive scraping agent
28 | type Source struct {
29 | timeTaken time.Duration
30 | errors int
31 | results int
32 | }
33 |
34 | // Run function returns all subdomains found with the service
35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
36 | results := make(chan subscraping.Result)
37 | s.errors = 0
38 | s.results = 0
39 |
40 | go func() {
41 | defer func(startTime time.Time) {
42 | s.timeTaken = time.Since(startTime)
43 | close(results)
44 | }(time.Now())
45 |
46 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain))
47 | if err != nil && resp == nil {
48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
49 | s.errors++
50 | session.DiscardHTTPResponse(resp)
51 | return
52 | }
53 |
54 | var response reconCloudResponse
55 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
56 | if err != nil {
57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
58 | s.errors++
59 | session.DiscardHTTPResponse(resp)
60 | return
61 | }
62 | session.DiscardHTTPResponse(resp)
63 |
64 | if len(response.CloudAssetsList) > 0 {
65 | for _, cloudAsset := range response.CloudAssetsList {
66 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}
67 | s.results++
68 | }
69 | }
70 | }()
71 |
72 | return results
73 | }
74 |
75 | // Name returns the name of the source
76 | func (s *Source) Name() string {
77 | return "reconcloud"
78 | }
79 |
80 | func (s *Source) IsDefault() bool {
81 | return true
82 | }
83 |
84 | func (s *Source) HasRecursiveSupport() bool {
85 | return true
86 | }
87 |
88 | func (s *Source) NeedsKey() bool {
89 | return false
90 | }
91 |
92 | func (s *Source) AddApiKeys(_ []string) {
93 | // no key needed
94 | }
95 |
96 | func (s *Source) Statistics() subscraping.Statistics {
97 | return subscraping.Statistics{
98 | Errors: s.errors,
99 | Results: s.results,
100 | TimeTaken: s.timeTaken,
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go:
--------------------------------------------------------------------------------
1 | // Package redhuntlabs logic
2 | package redhuntlabs
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "strings"
8 | "time"
9 |
10 | jsoniter "github.com/json-iterator/go"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | type Response struct {
16 | Subdomains []string `json:"subdomains"`
17 | Metadata ResponseMetadata `json:"metadata"`
18 | }
19 |
20 | type ResponseMetadata struct {
21 | ResultCount int `json:"result_count"`
22 | PageSize int `json:"page_size"`
23 | PageNumber int `json:"page_number"`
24 | }
25 |
26 | type Source struct {
27 | apiKeys []string
28 | timeTaken time.Duration
29 | errors int
30 | results int
31 | skipped bool
32 | }
33 |
34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
35 | results := make(chan subscraping.Result)
36 | s.errors = 0
37 | s.results = 0
38 | pageSize := 1000
39 | go func() {
40 | defer func(startTime time.Time) {
41 | s.timeTaken = time.Since(startTime)
42 | close(results)
43 | }(time.Now())
44 |
45 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
46 | if randomApiKey == "" || !strings.Contains(randomApiKey, ":") {
47 | s.skipped = true
48 | return
49 | }
50 |
51 | randomApiInfo := strings.Split(randomApiKey, ":")
52 | if len(randomApiInfo) != 3 {
53 | s.skipped = true
54 | return
55 | }
56 | baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1]
57 | requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"}
58 | getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize)
59 | resp, err := session.Get(ctx, getUrl, "", requestHeaders)
60 | if err != nil {
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)}
62 | session.DiscardHTTPResponse(resp)
63 | s.errors++
64 | return
65 | }
66 | var response Response
67 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
68 | if err != nil {
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
70 | session.DiscardHTTPResponse(resp)
71 | s.errors++
72 | return
73 | }
74 |
75 | session.DiscardHTTPResponse(resp)
76 | if response.Metadata.ResultCount > pageSize {
77 | totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize
78 | for page := 1; page <= totalPages; page++ {
79 | getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize)
80 | resp, err := session.Get(ctx, getUrl, "", requestHeaders)
81 | if err != nil {
82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)}
83 | session.DiscardHTTPResponse(resp)
84 | s.errors++
85 | return
86 | }
87 |
88 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
89 | if err != nil {
90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
91 | session.DiscardHTTPResponse(resp)
92 | s.errors++
93 | continue
94 | }
95 |
96 | session.DiscardHTTPResponse(resp)
97 |
98 | for _, subdomain := range response.Subdomains {
99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
100 | s.results++
101 | }
102 | }
103 | } else {
104 | for _, subdomain := range response.Subdomains {
105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
106 | s.results++
107 | }
108 | }
109 |
110 | }()
111 | return results
112 | }
113 |
114 | func (s *Source) Name() string {
115 | return "redhuntlabs"
116 | }
117 |
118 | func (s *Source) IsDefault() bool {
119 | return true
120 | }
121 |
122 | func (s *Source) HasRecursiveSupport() bool {
123 | return false
124 | }
125 |
126 | func (s *Source) NeedsKey() bool {
127 | return true
128 | }
129 |
130 | func (s *Source) AddApiKeys(keys []string) {
131 | s.apiKeys = keys
132 | }
133 |
134 | func (s *Source) Statistics() subscraping.Statistics {
135 | return subscraping.Statistics{
136 | Errors: s.errors,
137 | Results: s.results,
138 | TimeTaken: s.timeTaken,
139 | Skipped: s.skipped,
140 | }
141 | }
142 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/riddler/riddler.go:
--------------------------------------------------------------------------------
1 | // Package riddler logic
2 | package riddler
3 |
4 | import (
5 | "bufio"
6 | "context"
7 | "fmt"
8 | "time"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | // Source is the passive scraping agent
14 | type Source struct {
15 | timeTaken time.Duration
16 | errors int
17 | results int
18 | }
19 |
20 | // Run function returns all subdomains found with the service
21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
22 | results := make(chan subscraping.Result)
23 | s.errors = 0
24 | s.results = 0
25 |
26 | go func() {
27 | defer func(startTime time.Time) {
28 | s.timeTaken = time.Since(startTime)
29 | close(results)
30 | }(time.Now())
31 |
32 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain))
33 | if err != nil {
34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
35 | s.errors++
36 | session.DiscardHTTPResponse(resp)
37 | return
38 | }
39 |
40 | scanner := bufio.NewScanner(resp.Body)
41 | for scanner.Scan() {
42 | line := scanner.Text()
43 | if line == "" {
44 | continue
45 | }
46 | for _, subdomain := range session.Extractor.Extract(line) {
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
48 | s.results++
49 | }
50 | }
51 | session.DiscardHTTPResponse(resp)
52 | }()
53 |
54 | return results
55 | }
56 |
57 | // Name returns the name of the source
58 | func (s *Source) Name() string {
59 | return "riddler"
60 | }
61 |
62 | func (s *Source) IsDefault() bool {
63 | return true
64 | }
65 |
66 | func (s *Source) HasRecursiveSupport() bool {
67 | return false
68 | }
69 |
70 | func (s *Source) NeedsKey() bool {
71 | return false
72 | }
73 |
74 | func (s *Source) AddApiKeys(_ []string) {
75 | // no key needed
76 | }
77 |
78 | func (s *Source) Statistics() subscraping.Statistics {
79 | return subscraping.Statistics{
80 | Errors: s.errors,
81 | Results: s.results,
82 | TimeTaken: s.timeTaken,
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/robtex/robtext.go:
--------------------------------------------------------------------------------
1 | // Package robtex logic
2 | package robtex
3 |
4 | import (
5 | "bufio"
6 | "bytes"
7 | "context"
8 | "fmt"
9 | "time"
10 |
11 | jsoniter "github.com/json-iterator/go"
12 |
13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
14 | )
15 |
16 | const (
17 | addrRecord = "A"
18 | iPv6AddrRecord = "AAAA"
19 | baseURL = "https://proapi.robtex.com/pdns"
20 | )
21 |
22 | // Source is the passive scraping agent
23 | type Source struct {
24 | apiKeys []string
25 | timeTaken time.Duration
26 | errors int
27 | results int
28 | skipped bool
29 | }
30 |
31 | type result struct {
32 | Rrname string `json:"rrname"`
33 | Rrdata string `json:"rrdata"`
34 | Rrtype string `json:"rrtype"`
35 | }
36 |
37 | // Run function returns all subdomains found with the service
38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
39 | results := make(chan subscraping.Result)
40 | s.errors = 0
41 | s.results = 0
42 |
43 | go func() {
44 | defer func(startTime time.Time) {
45 | s.timeTaken = time.Since(startTime)
46 | close(results)
47 | }(time.Now())
48 |
49 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
50 | if randomApiKey == "" {
51 | s.skipped = true
52 | return
53 | }
54 |
55 | headers := map[string]string{"Content-Type": "application/x-ndjson"}
56 |
57 | ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, randomApiKey), headers)
58 | if err != nil {
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
60 | s.errors++
61 | return
62 | }
63 |
64 | for _, result := range ips {
65 | if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord {
66 | domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers)
67 | if err != nil {
68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
69 | s.errors++
70 | return
71 | }
72 | for _, result := range domains {
73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}
74 | s.results++
75 | }
76 | }
77 | }
78 | }()
79 |
80 | return results
81 | }
82 |
83 | func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) {
84 | var results []result
85 |
86 | resp, err := session.Get(ctx, targetURL, "", headers)
87 | if err != nil {
88 | session.DiscardHTTPResponse(resp)
89 | return results, err
90 | }
91 |
92 | defer session.DiscardHTTPResponse(resp)
93 |
94 | scanner := bufio.NewScanner(resp.Body)
95 | for scanner.Scan() {
96 | line := scanner.Text()
97 | if line == "" {
98 | continue
99 | }
100 | var response result
101 | err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response)
102 | if err != nil {
103 | return results, err
104 | }
105 |
106 | results = append(results, response)
107 | }
108 |
109 | return results, nil
110 | }
111 |
112 | // Name returns the name of the source
113 | func (s *Source) Name() string {
114 | return "robtex"
115 | }
116 |
117 | func (s *Source) IsDefault() bool {
118 | return true
119 | }
120 |
121 | func (s *Source) HasRecursiveSupport() bool {
122 | return false
123 | }
124 |
125 | func (s *Source) NeedsKey() bool {
126 | return true
127 | }
128 |
129 | func (s *Source) AddApiKeys(keys []string) {
130 | s.apiKeys = keys
131 | }
132 |
133 | func (s *Source) Statistics() subscraping.Statistics {
134 | return subscraping.Statistics{
135 | Errors: s.errors,
136 | Results: s.results,
137 | TimeTaken: s.timeTaken,
138 | Skipped: s.skipped,
139 | }
140 | }
141 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/rsecloud/rsecloud.go:
--------------------------------------------------------------------------------
1 | package rsecloud
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "time"
7 |
8 | jsoniter "github.com/json-iterator/go"
9 |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 | )
12 |
13 | type response struct {
14 | Count int `json:"count"`
15 | Data []string `json:"data"`
16 | Page int `json:"page"`
17 | PageSize int `json:"pagesize"`
18 | TotalPages int `json:"total_pages"`
19 | }
20 |
21 | // Source is the passive scraping agent
22 | type Source struct {
23 | apiKeys []string
24 | timeTaken time.Duration
25 | errors int
26 | results int
27 | skipped bool
28 | }
29 |
30 | // Run function returns all subdomains found with the service
31 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
32 | results := make(chan subscraping.Result)
33 | s.errors = 0
34 | s.results = 0
35 |
36 | go func() {
37 | defer func(startTime time.Time) {
38 | s.timeTaken = time.Since(startTime)
39 | close(results)
40 | }(time.Now())
41 |
42 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
43 | if randomApiKey == "" {
44 | s.skipped = true
45 | return
46 | }
47 |
48 | headers := map[string]string{"Content-Type": "application/json", "X-API-Key": randomApiKey}
49 |
50 | fetchSubdomains := func(endpoint string) {
51 | page := 1
52 | for {
53 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers)
54 | if err != nil {
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
56 | s.errors++
57 | session.DiscardHTTPResponse(resp)
58 | return
59 | }
60 |
61 | var rseCloudResponse response
62 | err = jsoniter.NewDecoder(resp.Body).Decode(&rseCloudResponse)
63 | session.DiscardHTTPResponse(resp)
64 | if err != nil {
65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
66 | s.errors++
67 | return
68 | }
69 |
70 | for _, subdomain := range rseCloudResponse.Data {
71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
72 | s.results++
73 | }
74 |
75 | if page >= rseCloudResponse.TotalPages {
76 | break
77 | }
78 | page++
79 | }
80 | }
81 |
82 | fetchSubdomains("active")
83 | fetchSubdomains("passive")
84 | }()
85 |
86 | return results
87 | }
88 |
89 | // Name returns the name of the source
90 | func (s *Source) Name() string {
91 | return "rsecloud"
92 | }
93 |
94 | func (s *Source) IsDefault() bool {
95 | return true
96 | }
97 |
98 | func (s *Source) HasRecursiveSupport() bool {
99 | return false
100 | }
101 |
102 | func (s *Source) NeedsKey() bool {
103 | return true
104 | }
105 |
106 | func (s *Source) AddApiKeys(keys []string) {
107 | s.apiKeys = keys
108 | }
109 |
110 | func (s *Source) Statistics() subscraping.Statistics {
111 | return subscraping.Statistics{
112 | Errors: s.errors,
113 | Results: s.results,
114 | TimeTaken: s.timeTaken,
115 | Skipped: s.skipped,
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/securitytrails/securitytrails.go:
--------------------------------------------------------------------------------
1 | // Package securitytrails logic
2 | package securitytrails
3 |
4 | import (
5 | "bytes"
6 | "context"
7 | "fmt"
8 | "net/http"
9 | "strings"
10 | "time"
11 |
12 | jsoniter "github.com/json-iterator/go"
13 |
14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
15 | "github.com/projectdiscovery/utils/ptr"
16 | )
17 |
18 | type response struct {
19 | Meta struct {
20 | ScrollID string `json:"scroll_id"`
21 | } `json:"meta"`
22 | Records []struct {
23 | Hostname string `json:"hostname"`
24 | } `json:"records"`
25 | Subdomains []string `json:"subdomains"`
26 | }
27 |
28 | // Source is the passive scraping agent
29 | type Source struct {
30 | apiKeys []string
31 | timeTaken time.Duration
32 | errors int
33 | results int
34 | skipped bool
35 | }
36 |
37 | // Run function returns all subdomains found with the service
38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
39 | results := make(chan subscraping.Result)
40 | s.errors = 0
41 | s.results = 0
42 |
43 | go func() {
44 | defer func(startTime time.Time) {
45 | s.timeTaken = time.Since(startTime)
46 | close(results)
47 | }(time.Now())
48 |
49 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
50 | if randomApiKey == "" {
51 | s.skipped = true
52 | return
53 | }
54 |
55 | var scrollId string
56 | headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey}
57 |
58 | for {
59 | var resp *http.Response
60 | var err error
61 |
62 | if scrollId == "" {
63 | var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain)
64 | resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "",
65 | headers, bytes.NewReader(requestBody))
66 | } else {
67 | resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers)
68 | }
69 |
70 | if err != nil && ptr.Safe(resp).StatusCode == 403 {
71 | resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers)
72 | }
73 |
74 | if err != nil {
75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
76 | s.errors++
77 | session.DiscardHTTPResponse(resp)
78 | return
79 | }
80 |
81 | var securityTrailsResponse response
82 | err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse)
83 | if err != nil {
84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
85 | s.errors++
86 | session.DiscardHTTPResponse(resp)
87 | return
88 | }
89 |
90 | session.DiscardHTTPResponse(resp)
91 |
92 | for _, record := range securityTrailsResponse.Records {
93 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
94 | s.results++
95 | }
96 |
97 | for _, subdomain := range securityTrailsResponse.Subdomains {
98 | if strings.HasSuffix(subdomain, ".") {
99 | subdomain += domain
100 | } else {
101 | subdomain = subdomain + "." + domain
102 | }
103 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
104 | s.results++
105 | }
106 |
107 | scrollId = securityTrailsResponse.Meta.ScrollID
108 |
109 | if scrollId == "" {
110 | break
111 | }
112 | }
113 | }()
114 |
115 | return results
116 | }
117 |
118 | // Name returns the name of the source
119 | func (s *Source) Name() string {
120 | return "securitytrails"
121 | }
122 |
123 | func (s *Source) IsDefault() bool {
124 | return true
125 | }
126 |
127 | func (s *Source) HasRecursiveSupport() bool {
128 | return true
129 | }
130 |
131 | func (s *Source) NeedsKey() bool {
132 | return true
133 | }
134 |
135 | func (s *Source) AddApiKeys(keys []string) {
136 | s.apiKeys = keys
137 | }
138 |
139 | func (s *Source) Statistics() subscraping.Statistics {
140 | return subscraping.Statistics{
141 | Errors: s.errors,
142 | Results: s.results,
143 | TimeTaken: s.timeTaken,
144 | Skipped: s.skipped,
145 | }
146 | }
147 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/shodan/shodan.go:
--------------------------------------------------------------------------------
1 | // Package shodan logic
2 | package shodan
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | // Source is the passive scraping agent
15 | type Source struct {
16 | apiKeys []string
17 | timeTaken time.Duration
18 | errors int
19 | results int
20 | skipped bool
21 | }
22 |
23 | type dnsdbLookupResponse struct {
24 | Domain string `json:"domain"`
25 | Subdomains []string `json:"subdomains"`
26 | Result int `json:"result"`
27 | Error string `json:"error"`
28 | More bool `json:"more"`
29 | }
30 |
31 | // Run function returns all subdomains found with the service
32 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
33 | results := make(chan subscraping.Result)
34 | s.errors = 0
35 | s.results = 0
36 |
37 | go func() {
38 | defer func(startTime time.Time) {
39 | s.timeTaken = time.Since(startTime)
40 | close(results)
41 | }(time.Now())
42 |
43 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
44 | if randomApiKey == "" {
45 | s.skipped = true
46 | return
47 | }
48 |
49 | page := 1
50 | for {
51 |
52 | searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page)
53 | resp, err := session.SimpleGet(ctx, searchURL)
54 | if err != nil {
55 | session.DiscardHTTPResponse(resp)
56 | return
57 | }
58 |
59 | defer session.DiscardHTTPResponse(resp)
60 |
61 | var response dnsdbLookupResponse
62 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
63 | if err != nil {
64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
65 | s.errors++
66 | return
67 | }
68 |
69 | if response.Error != "" {
70 | results <- subscraping.Result{
71 | Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error),
72 | }
73 | s.errors++
74 | return
75 | }
76 |
77 | for _, data := range response.Subdomains {
78 | value := fmt.Sprintf("%s.%s", data, response.Domain)
79 | results <- subscraping.Result{
80 | Source: s.Name(), Type: subscraping.Subdomain, Value: value,
81 | }
82 | s.results++
83 | }
84 |
85 | if !response.More {
86 | break
87 | }
88 | page++
89 | }
90 | }()
91 |
92 | return results
93 | }
94 |
95 | // Name returns the name of the source
96 | func (s *Source) Name() string {
97 | return "shodan"
98 | }
99 |
100 | func (s *Source) IsDefault() bool {
101 | return true
102 | }
103 |
104 | func (s *Source) HasRecursiveSupport() bool {
105 | return false
106 | }
107 |
108 | func (s *Source) NeedsKey() bool {
109 | return true
110 | }
111 |
112 | func (s *Source) AddApiKeys(keys []string) {
113 | s.apiKeys = keys
114 | }
115 |
116 | func (s *Source) Statistics() subscraping.Statistics {
117 | return subscraping.Statistics{
118 | Errors: s.errors,
119 | Results: s.results,
120 | TimeTaken: s.timeTaken,
121 | Skipped: s.skipped,
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/sitedossier/sitedossier.go:
--------------------------------------------------------------------------------
1 | // Package sitedossier logic
2 | package sitedossier
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "io"
8 | "net/http"
9 | "regexp"
10 | "time"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | // SleepRandIntn is the integer value to get the pseudo-random number
16 | // to sleep before find the next match
17 | const SleepRandIntn = 5
18 |
19 | var reNext = regexp.MustCompile(`<a href="([A-Za-z0-9/.]+)"><b>`)
20 |
21 | // Source is the passive scraping agent
22 | type Source struct {
23 | timeTaken time.Duration
24 | errors int
25 | results int
26 | }
27 |
28 | // Run function returns all subdomains found with the service
29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
30 | results := make(chan subscraping.Result)
31 | s.errors = 0
32 | s.results = 0
33 |
34 | go func() {
35 | defer func(startTime time.Time) {
36 | s.timeTaken = time.Since(startTime)
37 | close(results)
38 | }(time.Now())
39 |
40 | s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain), results)
41 | }()
42 |
43 | return results
44 | }
45 |
46 | func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, results chan subscraping.Result) {
47 | select {
48 | case <-ctx.Done():
49 | return
50 | default:
51 | }
52 |
53 | resp, err := session.SimpleGet(ctx, baseURL)
54 | isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound
55 | if err != nil && !isnotfound {
56 | results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
57 | s.errors++
58 | session.DiscardHTTPResponse(resp)
59 | return
60 | }
61 |
62 | body, err := io.ReadAll(resp.Body)
63 | if err != nil {
64 | results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
65 | s.errors++
66 | session.DiscardHTTPResponse(resp)
67 | return
68 | }
69 | session.DiscardHTTPResponse(resp)
70 |
71 | src := string(body)
72 | for _, subdomain := range session.Extractor.Extract(src) {
73 | results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}
74 | s.results++
75 | }
76 |
77 | match := reNext.FindStringSubmatch(src)
78 | if len(match) > 0 {
79 | s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com%s", match[1]), results)
80 | }
81 | }
82 |
83 | // Name returns the name of the source
84 | func (s *Source) Name() string {
85 | return "sitedossier"
86 | }
87 |
88 | func (s *Source) IsDefault() bool {
89 | return false
90 | }
91 |
92 | func (s *Source) HasRecursiveSupport() bool {
93 | return false
94 | }
95 |
96 | func (s *Source) NeedsKey() bool {
97 | return false
98 | }
99 |
100 | func (s *Source) AddApiKeys(_ []string) {
101 | // no key needed
102 | }
103 |
104 | func (s *Source) Statistics() subscraping.Statistics {
105 | return subscraping.Statistics{
106 | Errors: s.errors,
107 | Results: s.results,
108 | TimeTaken: s.timeTaken,
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/threatbook/threatbook.go:
--------------------------------------------------------------------------------
1 | // Package threatbook logic
2 | package threatbook
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "strconv"
8 | "time"
9 |
10 | jsoniter "github.com/json-iterator/go"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | type threatBookResponse struct {
16 | ResponseCode int64 `json:"response_code"`
17 | VerboseMsg string `json:"verbose_msg"`
18 | Data struct {
19 | Domain string `json:"domain"`
20 | SubDomains struct {
21 | Total string `json:"total"`
22 | Data []string `json:"data"`
23 | } `json:"sub_domains"`
24 | } `json:"data"`
25 | }
26 |
27 | // Source is the passive scraping agent
28 | type Source struct {
29 | apiKeys []string
30 | timeTaken time.Duration
31 | errors int
32 | results int
33 | skipped bool
34 | }
35 |
36 | // Run function returns all subdomains found with the service
37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
38 | results := make(chan subscraping.Result)
39 | s.errors = 0
40 | s.results = 0
41 |
42 | go func() {
43 | defer func(startTime time.Time) {
44 | s.timeTaken = time.Since(startTime)
45 | close(results)
46 | }(time.Now())
47 |
48 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
49 | if randomApiKey == "" {
50 | s.skipped = true
51 | return
52 | }
53 |
54 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain))
55 | if err != nil && resp == nil {
56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
57 | s.errors++
58 | session.DiscardHTTPResponse(resp)
59 | return
60 | }
61 |
62 | var response threatBookResponse
63 | err = jsoniter.NewDecoder(resp.Body).Decode(&response)
64 | if err != nil {
65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
66 | s.errors++
67 | session.DiscardHTTPResponse(resp)
68 | return
69 | }
70 | session.DiscardHTTPResponse(resp)
71 |
72 | if response.ResponseCode != 0 {
73 | results <- subscraping.Result{
74 | Source: s.Name(), Type: subscraping.Error,
75 | Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg),
76 | }
77 | s.errors++
78 | return
79 | }
80 |
81 | total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64)
82 | if err != nil {
83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
84 | s.errors++
85 | return
86 | }
87 |
88 | if total > 0 {
89 | for _, subdomain := range response.Data.SubDomains.Data {
90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
91 | s.results++
92 | }
93 | }
94 | }()
95 |
96 | return results
97 | }
98 |
99 | // Name returns the name of the source
100 | func (s *Source) Name() string {
101 | return "threatbook"
102 | }
103 |
104 | func (s *Source) IsDefault() bool {
105 | return false
106 | }
107 |
108 | func (s *Source) HasRecursiveSupport() bool {
109 | return false
110 | }
111 |
112 | func (s *Source) NeedsKey() bool {
113 | return true
114 | }
115 |
116 | func (s *Source) AddApiKeys(keys []string) {
117 | s.apiKeys = keys
118 | }
119 |
120 | func (s *Source) Statistics() subscraping.Statistics {
121 | return subscraping.Statistics{
122 | Errors: s.errors,
123 | Results: s.results,
124 | TimeTaken: s.timeTaken,
125 | Skipped: s.skipped,
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go:
--------------------------------------------------------------------------------
1 | package threatcrowd
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "io"
8 | "net/http"
9 | "time"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | // threatCrowdResponse represents the JSON response from the ThreatCrowd API.
15 | type threatCrowdResponse struct {
16 | ResponseCode string `json:"response_code"`
17 | Subdomains []string `json:"subdomains"`
18 | Undercount string `json:"undercount"`
19 | }
20 |
21 | // Source implements the subscraping.Source interface for ThreatCrowd.
22 | type Source struct {
23 | timeTaken time.Duration
24 | errors int
25 | results int
26 | }
27 |
28 | // Run queries the ThreatCrowd API for the given domain and returns found subdomains.
29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
30 | results := make(chan subscraping.Result)
31 | s.errors = 0
32 | s.results = 0
33 |
34 | go func(startTime time.Time) {
35 | defer func() {
36 | s.timeTaken = time.Since(startTime)
37 | close(results)
38 | }()
39 |
40 | url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)
41 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
42 | if err != nil {
43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
44 | s.errors++
45 | return
46 | }
47 |
48 | resp, err := session.Client.Do(req)
49 | if err != nil {
50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
51 | s.errors++
52 | return
53 | }
54 | defer func() {
55 | if err := resp.Body.Close(); err != nil {
56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
57 | s.errors++
58 | }
59 | }()
60 |
61 | if resp.StatusCode != http.StatusOK {
62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)}
63 | s.errors++
64 | return
65 | }
66 |
67 | body, err := io.ReadAll(resp.Body)
68 | if err != nil {
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
70 | s.errors++
71 | return
72 | }
73 |
74 | var tcResponse threatCrowdResponse
75 | if err := json.Unmarshal(body, &tcResponse); err != nil {
76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
77 | s.errors++
78 | return
79 | }
80 |
81 | for _, subdomain := range tcResponse.Subdomains {
82 | if subdomain != "" {
83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
84 | s.results++
85 | }
86 | }
87 | }(time.Now())
88 |
89 | return results
90 | }
91 |
92 | // Name returns the name of the source.
93 | func (s *Source) Name() string {
94 | return "threatcrowd"
95 | }
96 |
97 | // IsDefault indicates whether this source is enabled by default.
98 | func (s *Source) IsDefault() bool {
99 | return false
100 | }
101 |
102 | // HasRecursiveSupport indicates if the source supports recursive searches.
103 | func (s *Source) HasRecursiveSupport() bool {
104 | return false
105 | }
106 |
107 | // NeedsKey indicates if the source requires an API key.
108 | func (s *Source) NeedsKey() bool {
109 | return false
110 | }
111 |
112 | // AddApiKeys is a no-op since ThreatCrowd does not require an API key.
113 | func (s *Source) AddApiKeys(_ []string) {}
114 |
115 | // Statistics returns usage statistics.
116 | func (s *Source) Statistics() subscraping.Statistics {
117 | return subscraping.Statistics{
118 | Errors: s.errors,
119 | Results: s.results,
120 | TimeTaken: s.timeTaken,
121 | }
122 | }
123 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/threatminer/threatminer.go:
--------------------------------------------------------------------------------
1 | // Package threatminer logic
2 | package threatminer
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | type response struct {
15 | StatusCode string `json:"status_code"`
16 | StatusMessage string `json:"status_message"`
17 | Results []string `json:"results"`
18 | }
19 |
20 | // Source is the passive scraping agent
21 | type Source struct {
22 | timeTaken time.Duration
23 | errors int
24 | results int
25 | }
26 |
27 | // Run function returns all subdomains found with the service
28 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
29 | results := make(chan subscraping.Result)
30 | s.errors = 0
31 | s.results = 0
32 |
33 | go func() {
34 | defer func(startTime time.Time) {
35 | s.timeTaken = time.Since(startTime)
36 | close(results)
37 | }(time.Now())
38 |
39 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain))
40 | if err != nil {
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
42 | s.errors++
43 | session.DiscardHTTPResponse(resp)
44 | return
45 | }
46 |
47 | defer session.DiscardHTTPResponse(resp)
48 |
49 | var data response
50 | err = jsoniter.NewDecoder(resp.Body).Decode(&data)
51 | if err != nil {
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
53 | s.errors++
54 | return
55 | }
56 |
57 | for _, subdomain := range data.Results {
58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
59 | s.results++
60 | }
61 | }()
62 |
63 | return results
64 | }
65 |
66 | // Name returns the name of the source
67 | func (s *Source) Name() string {
68 | return "threatminer"
69 | }
70 |
71 | func (s *Source) IsDefault() bool {
72 | return true
73 | }
74 |
75 | func (s *Source) HasRecursiveSupport() bool {
76 | return false
77 | }
78 |
79 | func (s *Source) NeedsKey() bool {
80 | return false
81 | }
82 |
83 | func (s *Source) AddApiKeys(_ []string) {
84 | // no key needed
85 | }
86 |
87 | func (s *Source) Statistics() subscraping.Statistics {
88 | return subscraping.Statistics{
89 | Errors: s.errors,
90 | Results: s.results,
91 | TimeTaken: s.timeTaken,
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/virustotal/virustotal.go:
--------------------------------------------------------------------------------
1 | // Package virustotal logic
2 | package virustotal
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | type response struct {
15 | Data []Object `json:"data"`
16 | Meta Meta `json:"meta"`
17 | }
18 |
19 | type Object struct {
20 | Id string `json:"id"`
21 | }
22 |
23 | type Meta struct {
24 | Cursor string `json:"cursor"`
25 | }
26 |
27 | // Source is the passive scraping agent
28 | type Source struct {
29 | apiKeys []string
30 | timeTaken time.Duration
31 | errors int
32 | results int
33 | skipped bool
34 | }
35 |
36 | // Run function returns all subdomains found with the service
37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
38 | results := make(chan subscraping.Result)
39 | s.errors = 0
40 | s.results = 0
41 |
42 | go func() {
43 | defer func(startTime time.Time) {
44 | s.timeTaken = time.Since(startTime)
45 | close(results)
46 | }(time.Now())
47 |
48 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
49 | if randomApiKey == "" {
50 | return
51 | }
52 | var cursor = ""
53 | for {
54 | var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain)
55 | if cursor != "" {
56 | url = fmt.Sprintf("%s&cursor=%s", url, cursor)
57 | }
58 | resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey})
59 | if err != nil {
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 | s.errors++
62 | session.DiscardHTTPResponse(resp)
63 | return
64 | }
65 | defer func() {
66 | if err := resp.Body.Close(); err != nil {
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
68 | s.errors++
69 | }
70 | }()
71 |
72 | var data response
73 | err = jsoniter.NewDecoder(resp.Body).Decode(&data)
74 | if err != nil {
75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
76 | s.errors++
77 | return
78 | }
79 |
80 | for _, subdomain := range data.Data {
81 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}
82 | s.results++
83 | }
84 | cursor = data.Meta.Cursor
85 | if cursor == "" {
86 | break
87 | }
88 | }
89 | }()
90 |
91 | return results
92 | }
93 |
94 | // Name returns the name of the source
95 | func (s *Source) Name() string {
96 | return "virustotal"
97 | }
98 |
99 | func (s *Source) IsDefault() bool {
100 | return true
101 | }
102 |
103 | func (s *Source) HasRecursiveSupport() bool {
104 | return true
105 | }
106 |
107 | func (s *Source) NeedsKey() bool {
108 | return true
109 | }
110 |
111 | func (s *Source) AddApiKeys(keys []string) {
112 | s.apiKeys = keys
113 | }
114 |
115 | func (s *Source) Statistics() subscraping.Statistics {
116 | return subscraping.Statistics{
117 | Errors: s.errors,
118 | Results: s.results,
119 | TimeTaken: s.timeTaken,
120 | Skipped: s.skipped,
121 | }
122 | }
123 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go:
--------------------------------------------------------------------------------
1 | // Package waybackarchive logic
2 | package waybackarchive
3 |
4 | import (
5 | "bufio"
6 | "context"
7 | "fmt"
8 | "net/url"
9 | "strings"
10 | "time"
11 |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
13 | )
14 |
15 | // Source is the passive scraping agent
16 | type Source struct {
17 | timeTaken time.Duration
18 | errors int
19 | results int
20 | }
21 |
22 | // Run function returns all subdomains found with the service
23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
24 | results := make(chan subscraping.Result)
25 | s.errors = 0
26 | s.results = 0
27 |
28 | go func() {
29 | defer func(startTime time.Time) {
30 | s.timeTaken = time.Since(startTime)
31 | close(results)
32 | }(time.Now())
33 |
34 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain))
35 | if err != nil {
36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
37 | s.errors++
38 | session.DiscardHTTPResponse(resp)
39 | return
40 | }
41 |
42 | defer session.DiscardHTTPResponse(resp)
43 |
44 | scanner := bufio.NewScanner(resp.Body)
45 | for scanner.Scan() {
46 | line := scanner.Text()
47 | if line == "" {
48 | continue
49 | }
50 | line, _ = url.QueryUnescape(line)
51 | for _, subdomain := range session.Extractor.Extract(line) {
52 | // fix for triple encoded URL
53 | subdomain = strings.ToLower(subdomain)
54 | subdomain = strings.TrimPrefix(subdomain, "25")
55 | subdomain = strings.TrimPrefix(subdomain, "2f")
56 |
57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
58 | s.results++
59 | }
60 | }
61 | }()
62 |
63 | return results
64 | }
65 |
66 | // Name returns the name of the source
67 | func (s *Source) Name() string {
68 | return "waybackarchive"
69 | }
70 |
71 | func (s *Source) IsDefault() bool {
72 | return false
73 | }
74 |
75 | func (s *Source) HasRecursiveSupport() bool {
76 | return false
77 | }
78 |
79 | func (s *Source) NeedsKey() bool {
80 | return false
81 | }
82 |
83 | func (s *Source) AddApiKeys(_ []string) {
84 | // no key needed
85 | }
86 |
87 | func (s *Source) Statistics() subscraping.Statistics {
88 | return subscraping.Statistics{
89 | Errors: s.errors,
90 | Results: s.results,
91 | TimeTaken: s.timeTaken,
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go:
--------------------------------------------------------------------------------
1 | // Package whoisxmlapi logic
2 | package whoisxmlapi
3 |
4 | import (
5 | "context"
6 | "fmt"
7 | "time"
8 |
9 | jsoniter "github.com/json-iterator/go"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | type response struct {
15 | Search string `json:"search"`
16 | Result Result `json:"result"`
17 | }
18 |
19 | type Result struct {
20 | Count int `json:"count"`
21 | Records []Record `json:"records"`
22 | }
23 |
24 | type Record struct {
25 | Domain string `json:"domain"`
26 | FirstSeen int `json:"firstSeen"`
27 | LastSeen int `json:"lastSeen"`
28 | }
29 |
30 | // Source is the passive scraping agent
31 | type Source struct {
32 | apiKeys []string
33 | timeTaken time.Duration
34 | errors int
35 | results int
36 | skipped bool
37 | }
38 |
39 | // Run function returns all subdomains found with the service
40 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
41 | results := make(chan subscraping.Result)
42 | s.errors = 0
43 | s.results = 0
44 |
45 | go func() {
46 | defer func(startTime time.Time) {
47 | s.timeTaken = time.Since(startTime)
48 | close(results)
49 | }(time.Now())
50 |
51 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
52 | if randomApiKey == "" {
53 | s.skipped = true
54 | return
55 | }
56 |
57 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain))
58 | if err != nil {
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
60 | s.errors++
61 | session.DiscardHTTPResponse(resp)
62 | return
63 | }
64 |
65 | var data response
66 | err = jsoniter.NewDecoder(resp.Body).Decode(&data)
67 | if err != nil {
68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
69 | s.errors++
70 | session.DiscardHTTPResponse(resp)
71 | return
72 | }
73 |
74 | session.DiscardHTTPResponse(resp)
75 |
76 | for _, record := range data.Result.Records {
77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}
78 | s.results++
79 | }
80 | }()
81 |
82 | return results
83 | }
84 |
85 | // Name returns the name of the source
86 | func (s *Source) Name() string {
87 | return "whoisxmlapi"
88 | }
89 |
90 | func (s *Source) IsDefault() bool {
91 | return true
92 | }
93 |
94 | func (s *Source) HasRecursiveSupport() bool {
95 | return false
96 | }
97 |
98 | func (s *Source) NeedsKey() bool {
99 | return true
100 | }
101 |
102 | func (s *Source) AddApiKeys(keys []string) {
103 | s.apiKeys = keys
104 | }
105 |
106 | func (s *Source) Statistics() subscraping.Statistics {
107 | return subscraping.Statistics{
108 | Errors: s.errors,
109 | Results: s.results,
110 | TimeTaken: s.timeTaken,
111 | Skipped: s.skipped,
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go:
--------------------------------------------------------------------------------
1 | package zoomeyeapi
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "net/http"
8 | "strings"
9 | "time"
10 |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 | )
13 |
14 | // search results
15 | type zoomeyeResults struct {
16 | Status int `json:"status"`
17 | Total int `json:"total"`
18 | List []struct {
19 | Name string `json:"name"`
20 | Ip []string `json:"ip"`
21 | } `json:"list"`
22 | }
23 |
24 | // Source is the passive scraping agent
25 | type Source struct {
26 | apiKeys []string
27 | timeTaken time.Duration
28 | errors int
29 | results int
30 | skipped bool
31 | }
32 |
33 | // Run function returns all subdomains found with the service
34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
35 | results := make(chan subscraping.Result)
36 | s.errors = 0
37 | s.results = 0
38 |
39 | go func() {
40 | defer func(startTime time.Time) {
41 | s.timeTaken = time.Since(startTime)
42 | close(results)
43 | }(time.Now())
44 |
45 | randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
46 | if randomApiKey == "" {
47 | s.skipped = true
48 | return
49 | }
50 |
51 | randomApiInfo := strings.Split(randomApiKey, ":")
52 | if len(randomApiInfo) != 2 {
53 | s.skipped = true
54 | return
55 | }
56 | host := randomApiInfo[0]
57 | apiKey := randomApiInfo[1]
58 |
59 | headers := map[string]string{
60 | "API-KEY": apiKey,
61 | "Accept": "application/json",
62 | "Content-Type": "application/json",
63 | }
64 | var pages = 1
65 | for currentPage := 1; currentPage <= pages; currentPage++ {
66 | api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage)
67 | resp, err := session.Get(ctx, api, "", headers)
68 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
69 | if err != nil {
70 | if !isForbidden {
71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
72 | s.errors++
73 | session.DiscardHTTPResponse(resp)
74 | }
75 | return
76 | }
77 |
78 | var res zoomeyeResults
79 | err = json.NewDecoder(resp.Body).Decode(&res)
80 |
81 | if err != nil {
82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
83 | s.errors++
84 | _ = resp.Body.Close()
85 | return
86 | }
87 | _ = resp.Body.Close()
88 | pages = int(res.Total/1000) + 1
89 | for _, r := range res.List {
90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}
91 | s.results++
92 | }
93 | }
94 | }()
95 |
96 | return results
97 | }
98 |
99 | // Name returns the name of the source
100 | func (s *Source) Name() string {
101 | return "zoomeyeapi"
102 | }
103 |
104 | func (s *Source) IsDefault() bool {
105 | return false
106 | }
107 |
108 | func (s *Source) HasRecursiveSupport() bool {
109 | return false
110 | }
111 |
112 | func (s *Source) NeedsKey() bool {
113 | return true
114 | }
115 |
116 | func (s *Source) AddApiKeys(keys []string) {
117 | s.apiKeys = keys
118 | }
119 |
120 | func (s *Source) Statistics() subscraping.Statistics {
121 | return subscraping.Statistics{
122 | Errors: s.errors,
123 | Results: s.results,
124 | TimeTaken: s.timeTaken,
125 | Skipped: s.skipped,
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/types.go:
--------------------------------------------------------------------------------
1 | package subscraping
2 |
3 | import (
4 | "context"
5 | "net/http"
6 | "time"
7 |
8 | "github.com/projectdiscovery/ratelimit"
9 | mapsutil "github.com/projectdiscovery/utils/maps"
10 | )
11 |
12 | type CtxArg string
13 |
14 | const (
15 | CtxSourceArg CtxArg = "source"
16 | )
17 |
18 | type CustomRateLimit struct {
19 | Custom mapsutil.SyncLockMap[string, uint]
20 | }
21 |
22 | // BasicAuth request's Authorization header
23 | type BasicAuth struct {
24 | Username string
25 | Password string
26 | }
27 |
28 | // Statistics contains statistics about the scraping process
29 | type Statistics struct {
30 | TimeTaken time.Duration
31 | Errors int
32 | Results int
33 | Skipped bool
34 | }
35 |
36 | // Source is an interface inherited by each passive source
37 | type Source interface {
38 | // Run takes a domain as argument and a session object
39 | // which contains the extractor for subdomains, http client
40 | // and other stuff.
41 | Run(context.Context, string, *Session) <-chan Result
42 |
43 | // Name returns the name of the source. It is preferred to use lower case names.
44 | Name() string
45 |
46 | // IsDefault returns true if the current source should be
47 | // used as part of the default execution.
48 | IsDefault() bool
49 |
50 | // HasRecursiveSupport returns true if the current source
51 | // accepts subdomains (e.g. subdomain.domain.tld),
52 | // not just root domains.
53 | HasRecursiveSupport() bool
54 |
55 | // NeedsKey returns true if the source requires an API key
56 | NeedsKey() bool
57 |
58 | AddApiKeys([]string)
59 |
60 | // Statistics returns the scrapping statistics for the source
61 | Statistics() Statistics
62 | }
63 |
64 | // SubdomainExtractor is an interface that defines the contract for subdomain extraction.
65 | type SubdomainExtractor interface {
66 | Extract(text string) []string
67 | }
68 |
69 | // Session is the option passed to the source, an option is created
70 | // uniquely for each source.
71 | type Session struct {
72 | //SubdomainExtractor
73 | Extractor SubdomainExtractor
74 | // Client is the current http client
75 | Client *http.Client
76 | // Rate limit instance
77 | MultiRateLimiter *ratelimit.MultiLimiter
78 | }
79 |
80 | // Result is a result structure returned by a source
81 | type Result struct {
82 | Type ResultType
83 | Source string
84 | Value string
85 | Error error
86 | }
87 |
88 | // ResultType is the type of result returned by the source
89 | type ResultType int
90 |
91 | // Types of results returned by the source
92 | const (
93 | Subdomain ResultType = iota
94 | Error
95 | )
96 |
--------------------------------------------------------------------------------
/v2/pkg/subscraping/utils.go:
--------------------------------------------------------------------------------
1 | package subscraping
2 |
3 | import (
4 | "math/rand"
5 | "strings"
6 |
7 | "github.com/projectdiscovery/gologger"
8 | )
9 |
10 | const MultipleKeyPartsLength = 2
11 |
12 | func PickRandom[T any](v []T, sourceName string) T {
13 | var result T
14 | length := len(v)
15 | if length == 0 {
16 | gologger.Debug().Msgf("Cannot use the %s source because there was no API key/secret defined for it.", sourceName)
17 | return result
18 | }
19 | return v[rand.Intn(length)]
20 | }
21 |
22 | func CreateApiKeys[T any](keys []string, provider func(k, v string) T) []T {
23 | var result []T
24 | for _, key := range keys {
25 | if keyPartA, keyPartB, ok := createMultiPartKey(key); ok {
26 | result = append(result, provider(keyPartA, keyPartB))
27 | }
28 | }
29 | return result
30 | }
31 |
32 | func createMultiPartKey(key string) (keyPartA, keyPartB string, ok bool) {
33 | parts := strings.Split(key, ":")
34 | ok = len(parts) == MultipleKeyPartsLength
35 |
36 | if ok {
37 | keyPartA = parts[0]
38 | keyPartB = parts[1]
39 | }
40 |
41 | return
42 | }
43 |
--------------------------------------------------------------------------------
/v2/pkg/testutils/integration.go:
--------------------------------------------------------------------------------
1 | package testutils
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "os/exec"
7 | "strings"
8 | )
9 |
10 | func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]string, error) {
11 | cmd := exec.Command("bash", "-c")
12 | cmdLine := fmt.Sprintf("echo %s | %s", domain, "./subfinder ")
13 | cmdLine += strings.Join(extra, " ")
14 | cmd.Args = append(cmd.Args, cmdLine)
15 | if debug {
16 | cmd.Args = append(cmd.Args, "-v")
17 | cmd.Stderr = os.Stderr
18 | fmt.Println(cmd.String())
19 | } else {
20 | cmd.Args = append(cmd.Args, "-silent")
21 | }
22 | data, err := cmd.Output()
23 | if debug {
24 | fmt.Println(string(data))
25 | }
26 | if err != nil {
27 | return nil, err
28 | }
29 | var parts []string
30 | items := strings.SplitSeq(string(data), "\n")
31 | for i := range items {
32 | if i != "" {
33 | parts = append(parts, i)
34 | }
35 | }
36 | return parts, nil
37 | }
38 |
39 | // TestCase is a single integration test case
40 | type TestCase interface {
41 | Execute() error
42 | }
43 |
--------------------------------------------------------------------------------