├── .env.example ├── .github └── workflows │ └── test.yml ├── .gitignore ├── .rubocop.yml ├── Dockerfile ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── bin └── main.rb ├── config └── exclusions.yml ├── libs ├── db │ └── db.rb ├── platforms │ ├── bugcrowd │ │ ├── auth.rb │ │ ├── programs.rb │ │ └── scopes.rb │ ├── config.rb │ ├── hackerone │ │ ├── programs.rb │ │ └── scopes.rb │ ├── immunefi │ │ ├── programs.rb │ │ └── scopes.rb │ ├── intigriti │ │ ├── programs.rb │ │ └── scopes.rb │ └── yeswehack │ │ ├── auth.rb │ │ ├── programs.rb │ │ └── scopes.rb ├── scopes_extractor.rb └── utilities │ ├── http_client.rb │ ├── logger.rb │ ├── normalizer │ ├── bugcrowd.rb │ ├── hackerone.rb │ ├── intigriti.rb │ ├── normalizer.rb │ └── yeswehack.rb │ ├── notifier.rb │ ├── parser.rb │ └── scopes_comparator.rb └── spec ├── spec_helper.rb └── utilities ├── normalizer ├── bugcrowd_spec.rb ├── hackerone_spec.rb ├── intigriti_spec.rb └── yeswehack_spec.rb └── parser_spec.rb /.env.example: -------------------------------------------------------------------------------- 1 | API_MODE=true 2 | API_KEY="" 3 | 4 | AUTO_SYNC=false 5 | SYNC_DELAY=10800 6 | 7 | HISTORY_RETENTION_DAYS=30 8 | 9 | YWH_SYNC=false 10 | YWH_EMAIL='' 11 | YWH_PWD='' 12 | YWH_OTP='' 13 | 14 | INTIGRITI_SYNC=false 15 | INTIGRITI_TOKEN='' 16 | 17 | H1_SYNC=false 18 | H1_USERNAME='' 19 | H1_TOKEN='' 20 | 21 | BC_SYNC=false 22 | BC_EMAIL='' 23 | BC_PWD='' 24 | BC_OTP='' 25 | 26 | IMMUNEFI_SYNC=false 27 | 28 | # web,cidr,mobile,other,executable,hardware,iot,network,ai,device,blockchain,contracts,source_code 29 | NOTIFY_CATEGORIES=all 30 | 31 | DISCORD_WEBHOOK='https://discord.com/api/webhooks/xxx/xxx' 32 | DISCORD_LOGS_WEBHOOK='https://discord.com/api/webhooks/xxx/xxx' 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: ["*"] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | ruby-version: ["3.4.2"] 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | - name: Set up Ruby ${{ matrix.ruby-version }} 21 | uses: ruby/setup-ruby@v1 22 | with: 23 | ruby-version: ${{ matrix.ruby-version }} 24 | bundler-cache: true 25 | 26 | - name: Install dependencies 27 | run: bundle install 28 | 29 | - name: Setup Code Climate test-reporter 30 | run: | 31 | curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 32 | chmod +x ./cc-test-reporter 33 | ./cc-test-reporter before-build 34 | 35 | - name: Run tests 36 | run: bundle exec rake 37 | 38 | - name: Rename coverage file 39 | run: mv coverage/ScopesExtractor.lcov coverage/lcov.info 40 | if: success() 41 | 42 | - name: Publish code coverage 43 | run: | 44 | ./cc-test-reporter after-build -r ${{secrets.CC_TEST_REPORTER_ID}} -t lcov 45 | if: success() 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .env 3 | 4 | .idea/ 5 | extract.json 6 | Gemfile.lock 7 | libs/db/db.json 8 | /coverage 9 | libs/db/history.json 10 | libs/db/cookies.txt 11 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | NewCops: enable 3 | 4 | Metrics/BlockLength: 5 | Exclude: 6 | - "spec/**/*" 7 | 8 | Metrics/MethodLength: 9 | Max: 15 10 | Exclude: 11 | - "libs/platforms/config.rb" 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 ruby:3.4.2 2 | 3 | WORKDIR /app 4 | COPY . . 5 | COPY Gemfile Gemfile 6 | 7 | RUN bundle install 8 | 9 | CMD ruby bin/main.rb 10 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | gem 'base64' 6 | gem 'colorize' 7 | gem 'dotenv' 8 | gem 'nokogiri' 9 | gem 'rotp' 10 | gem 'typhoeus' 11 | gem 'webrick' 12 | 13 | group :development, :test do 14 | gem 'rake', '~> 13.0' 15 | gem 'rspec', '~> 3.12' 16 | gem 'simplecov', '~> 0.22.0', require: false 17 | gem 'simplecov-lcov', require: false 18 | gem 'webmock' 19 | end 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 JoMar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Image](https://github.com/user-attachments/assets/8fa9dd2a-04c8-48d4-a0d7-6057c102436c) 2 | 3 | A tool for monitoring bug bounty programs across multiple platforms to track scope changes. 4 | 5 | [![Ruby](https://img.shields.io/badge/Ruby-3.4.2-red.svg)](https://www.ruby-lang.org/en/) 6 | [![Docker](https://img.shields.io/badge/Docker-Supported-blue.svg)](https://www.docker.com/) 7 | [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) 8 | [![codeclimate](https://api.codeclimate.com/v1/badges/713b3c783fe46abaca0e/maintainability)](https://codeclimate.com/github/JoshuaMart/ScopesExtractor/maintainability/) 9 | 10 | ## 📖 Overview 11 | 12 | Scopes Extractor is a Ruby application that monitors bug bounty programs. It tracks changes to program scopes (additions and removals) and sends notifications through Discord webhooks. The tool can be run in classic mode or API mode for querying the latest data. 13 | 14 | ## ✨ Features 15 | 16 | - 🔍 Monitors multiple bug bounty platforms (YesWeHack, Immunefi, Hackerone & Bugcrowd) 17 | - 🔄 Detects changes in program scopes 18 | - 📏 Normalizes scope formats for better consistency (e.g., domain.(tld|xyz) becomes domain.tld and domain.xyz) 19 | - 🚨 Sends notifications to Discord webhooks 20 | - 🔌 Offers an API mode for querying data 21 | - 🔄 Supports automatic synchronization with configurable intervals 22 | - 🔐 Authentication with platforms including OTP support 23 | - 💾 Persistent storage of program data in JSON format 24 | - 📊 Historical tracking of changes with retention policy 25 | 26 | ## 🛠️ Installation 27 | 28 | ### Prerequisites 29 | 30 | - Docker (recommended) or Ruby 3.4.2 31 | 32 | ### Setup 33 | 34 | 1. Clone the repository: 35 | ```bash 36 | git clone https://github.com/JoshuaMart/ScopesExtractor 37 | cd ScopesExtractor 38 | ``` 39 | 40 | 2. Create the environment file: 41 | ```bash 42 | cp .env.example .env 43 | ``` 44 | 45 | 3. Configure your `.env` file with: 46 | - YesWeHack, Intigriti, Hackerone and Bugcrowd credentials (if applicable) 47 | - Discord webhook URLs 48 | - API settings 49 | - Synchronization options 50 | - History retention policy 51 | 52 | 4. Build the Docker image: 53 | ```bash 54 | docker build . -t scopes 55 | ``` 56 | 57 | ## 🚀 Usage 58 | 59 | ### Classic Mode 60 | 61 | Run the application in classic mode (no API): 62 | 63 | ```bash 64 | docker run --mount type=bind,source="$(pwd)/libs/db/db.json",target=/app/libs/db/db.json --mount type=bind,source="$(pwd)/libs/db/history.json",target=/app/libs/db/history.json scopes 65 | ``` 66 | 67 | ### API Mode 68 | 69 | Run the application in API mode to expose HTTP endpoints for querying the data: 70 | 71 | ```bash 72 | docker run -p 4567:4567 --mount type=bind,source="$(pwd)/libs/db/db.json",target=/app/libs/db/db.json --mount type=bind,source="$(pwd)/libs/db/history.json",target=/app/libs/db/history.json scopes 73 | ``` 74 | 75 | When in API mode, you can query the data by sending a request to the endpoint with your configured API key: 76 | 77 | ```bash 78 | # Get current program data 79 | curl -H "X-API-Key: your_api_key_here" http://localhost:4567 80 | 81 | # Get recent changes (last 48 hours by default) 82 | curl -H "X-API-Key: your_api_key_here" http://localhost:4567/changes 83 | 84 | # Get changes from the last 24 hours 85 | curl -H "X-API-Key: your_api_key_here" "http://localhost:4567/changes?hours=24" 86 | 87 | # Filter changes by platform 88 | curl -H "X-API-Key: your_api_key_here" "http://localhost:4567/changes?platform=YesWeHack" 89 | 90 | # Filter by change type (add_program, remove_program, add_scope, remove_scope) 91 | curl -H "X-API-Key: your_api_key_here" "http://localhost:4567/changes?type=add_scope" 92 | 93 | # Filter by program name 94 | curl -H "X-API-Key: your_api_key_here" "http://localhost:4567/changes?program=ProgramName" 95 | 96 | # Filter by category 97 | curl -H "X-API-Key: your_api_key_here" "http://localhost:4567/changes?category=web" 98 | 99 | # Combine filters 100 | curl -H "X-API-Key: your_api_key_here" "http://localhost:4567/changes?hours=72&platform=Hackerone&type=add_scope" 101 | ``` 102 | 103 | ## ⚙️ Configuration 104 | 105 | ### Environment Variables 106 | 107 | | Variable | Description | Default | 108 | |----------|-------------|---------| 109 | | `API_MODE` | Enable/disable API mode | `false` | 110 | | `API_KEY` | API key for authentication | `""` | 111 | | `AUTO_SYNC` | Enable/disable automatic synchronization | `false` | 112 | | `SYNC_DELAY` | Delay between synchronizations (in seconds) | `10800` | 113 | | `HISTORY_RETENTION_DAYS` | Number of days to retain change history | `30` | 114 | | `YWH_SYNC` | Enable YesWeHack synchronization | `false` | 115 | | `YWH_EMAIL` | YesWeHack email | `""` | 116 | | `YWH_PWD` | YesWeHack password | `""` | 117 | | `YWH_OTP` | YesWeHack OTP secret | `""` | 118 | | `INTIGRITI_SYNC` | Enable Intigriti synchronization | `false` | 119 | | `INTIGRITI_TOKEN` | Intigriti API Token | `""` | 120 | | `H1_SYNC` | Enable Hackerone synchronization | `false` | 121 | | `H1_USERNAME` | Hackerone username | `""` | 122 | | `H1_TOKEN` | Hackerone API Token | `""` | 123 | | `BC_SYNC` | Enable Bugcrowd synchronization | `false` | 124 | | `BC_EMAIL` | Bugcrowd email | `""` | 125 | | `BC_PWD` | Bugcrowd password | `""` | 126 | | `BC_OTP` | Bugcrowd OTP secret | `""` | 127 | | `IMMUNEFI_SYNC` | Enable Immunefi synchronization | `false` | 128 | | `NOTIFY_CATEGORIES` | Scopes categories for which notifications are sent | `all` | 129 | | `DISCORD_WEBHOOK` | Discord webhook URL for program notifications | `""` | 130 | | `DISCORD_LOGS_WEBHOOK` | Discord webhook URL for log notifications | `""` | 131 | 132 | ### 📊 Exclusions 133 | 134 | You can configure pattern exclusions in `config/exclusions.yml` to filter out specific scopes. 135 | 136 | ## ✋ FAQ 137 | 138 |
139 | Some programs are missing 140 | 141 | VDPs and scopes without bounty not included 142 |
143 | 144 |
145 | Intigriti - Failed to fetch program ... 403 146 | 147 | Programs must be manually accepted on the Intigriti website in order to be able to consult them. 148 |
149 | 150 |
151 | Error : Invalid OTP code 152 | 153 | The most likely reason is that your server's time is not correct, so the generated OTP code is not correct either. 154 |
155 | 156 |
157 | Change History Informations 158 | 159 | ScopesExtractor now tracks all changes (program and scope additions/removals) with timestamps. This history is automatically managed with a configurable retention policy to avoid excessive growth. By default, changes are kept for 30 days. 160 | 161 | You can query recent changes through the API (only) to see what has changed in the last few hours or days, which is useful for keeping track of bug bounty program changes even if you missed the Discord notifications. 162 | 163 | The changes reflect what is detected by ScopesExtractor (addition/removal of scopes and programs) and not the modifications indicated directly on the program page of each platform. 164 |
165 | 166 | ## 📜 License 167 | 168 | This project is open-source and available under the MIT License. 169 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rspec/core/rake_task' 4 | 5 | RSpec::Core::RakeTask.new(:spec) do |task| 6 | task.rspec_opts = '--format documentation' 7 | end 8 | 9 | task default: :spec 10 | -------------------------------------------------------------------------------- /bin/main.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require_relative '../libs/scopes_extractor/' 5 | 6 | extractor = ScopesExtractor::Extract.new 7 | extractor.run 8 | -------------------------------------------------------------------------------- /config/exclusions.yml: -------------------------------------------------------------------------------- 1 | exclusions: 2 | # Bugcrowd 3 | - "█" 4 | -------------------------------------------------------------------------------- /libs/db/db.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'json' 4 | require 'fileutils' 5 | require 'time' 6 | 7 | module ScopesExtractor 8 | # DB module provides a simple flat-file JSON database for storing and retrieving program data 9 | module DB 10 | # Path to the JSON database file 11 | DB_FILE = File.join(__dir__, 'db.json') 12 | # Path to the JSON history file 13 | HISTORY_FILE = File.join(__dir__, 'history.json') 14 | 15 | # Loads program data from the JSON database file 16 | # @return [Hash] Program data from the database, empty hash if the file doesn't exist or is invalid 17 | def self.load 18 | return {} unless File.exist?(DB_FILE) 19 | 20 | file_content = File.read(DB_FILE) 21 | JSON.parse(file_content) 22 | rescue JSON::ParserError 23 | {} 24 | end 25 | 26 | # Saves program data to the JSON database file 27 | # @param data [Hash] Program data to save 28 | # @return [Integer] Number of bytes written to the file 29 | def self.save(data) 30 | File.write(DB_FILE, JSON.pretty_generate(data)) 31 | end 32 | 33 | # Loads change history from the JSON history file 34 | # @return [Array] Array of change history entries, empty array if the file doesn't exist or is invalid 35 | def self.load_history 36 | return [] unless File.exist?(HISTORY_FILE) 37 | 38 | file_content = File.read(HISTORY_FILE) 39 | JSON.parse(file_content) 40 | rescue JSON::ParserError 41 | [] 42 | end 43 | 44 | # Saves a change to the history file 45 | # @param platform [String] Platform name (e.g., 'YesWeHack') 46 | # @param program [String] Program title 47 | # @param change_type [String] Type of change ('add_program', 'remove_program', 'add_scope', 'remove_scope') 48 | # @param scope_type [String, nil] Scope type ('in' or 'out') for scope changes, nil for program changes 49 | # @param category [String, nil] Category for scope changes, nil for program changes 50 | # @param value [String] Value of scope or program name 51 | # @return [Integer] Number of bytes written to the file 52 | def self.save_change(platform, program, change_type, scope_type, category, value) 53 | history = load_history 54 | 55 | # Create a new entry 56 | entry = { 57 | 'timestamp' => Time.now.utc.iso8601, 58 | 'platform' => platform, 59 | 'program' => program, 60 | 'change_type' => change_type, 61 | 'scope_type' => scope_type, 62 | 'category' => category, 63 | 'value' => value 64 | } 65 | 66 | history << entry 67 | 68 | # Clean up old entries based on retention policy 69 | retention_days = Config.load.dig(:history, :retention_days) || 30 70 | cutoff_time = (Time.now.utc - (retention_days * 24 * 60 * 60)).iso8601 71 | history = history.select { |h| h['timestamp'] >= cutoff_time } 72 | 73 | File.write(HISTORY_FILE, JSON.pretty_generate(history)) 74 | end 75 | 76 | # Gets recent changes from the history file 77 | # @param hours [Integer] Number of hours to look back 78 | # @param filters [Hash] Optional filters for the changes (platform, program, change_type) 79 | # @return [Array] Array of recent changes matching the criteria 80 | def self.get_recent_changes(hours = 48, filters = {}) 81 | history = load_history 82 | cutoff_time = (Time.now.utc - (hours * 60 * 60)).iso8601 83 | 84 | # Filter by time first 85 | filtered = history.select { |entry| entry['timestamp'] >= cutoff_time } 86 | 87 | # Apply additional filters if specified 88 | filters.each do |key, value| 89 | filtered = filtered.select { |entry| entry[key.to_s] == value } 90 | end 91 | 92 | filtered 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /libs/platforms/bugcrowd/auth.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # Bugcrowd platform authentication utilities 5 | module Bugcrowd 6 | BASE_URL = 'https://identity.bugcrowd.com' 7 | DASHBOARD_URL = '/dashboard' 8 | 9 | # Authenticates with Bugcrowd 10 | # @param config [Hash] Configuration containing email and password 11 | # @return [Boolean] True if authentication is successful, false otherwise 12 | def self.authenticate(config) 13 | HttpClient.clear_cookie_jar 14 | url = "#{BASE_URL}/login?user_hint=researcher&returnTo=#{DASHBOARD_URL}" 15 | resp = HttpClient.get(url, { follow_location: true }) 16 | return { error: login_error(resp) } unless valid_response?(resp, 200) 17 | return { success: true } if authenticated?(resp) 18 | 19 | csrf = extract_csrf(resp) 20 | return { error: "No Login CSRF - #{resp.code}" } unless csrf 21 | 22 | response = login(config, csrf) 23 | return response if response[:error] 24 | 25 | check_authentication_success(response[:redirect_to]) 26 | end 27 | 28 | # Build error message for login page access request 29 | # @return [String] 30 | def self.login_error(resp) 31 | message = "Invalid base login response - #{resp.code}" 32 | message += "\n\nResponse Headers:```\n#{resp.headers}\n```" 33 | message 34 | end 35 | 36 | # Handles login request 37 | # @param config [Hash] Configuration containing email and password 38 | # @param csrf [String] CSRF token 39 | # @return [String, nil] Redirect URL if login successful, nil otherwise 40 | def self.login(config, csrf) 41 | options = prepare_request(config, csrf, false) 42 | resp = HttpClient.post("#{BASE_URL}/login", options) 43 | return { error: 'Invalid login or password' } unless valid_response?(resp, 422) 44 | 45 | options = prepare_request(config, csrf, true) 46 | resp = HttpClient.post("#{BASE_URL}/auth/otp-challenge", options) 47 | return { error: 'Invalid OTP code' } unless valid_response?(resp, 200) 48 | 49 | body = Parser.json_parse(resp.body) 50 | { redirect_to: body['redirect_to'] } 51 | end 52 | 53 | # Prepare request options for login 54 | # @param config [Hash] Configuration containing email and password 55 | # @param with_otp [Boolean] body request with or without otp_code 56 | # @return [Hash] 57 | def self.prepare_request(config, csrf, with_otp) 58 | { 59 | headers: { 'X-Csrf-Token' => csrf, 'Origin' => 'https://identity.bugcrowd.com' }, 60 | body: prepare_body(config, with_otp) 61 | } 62 | end 63 | 64 | # Prepare request body for login 65 | # @param config [Hash] Configuration containing email and password 66 | # @param with_otp [Boolean] body request with or without otp_code 67 | # @return [String] Encoded request body 68 | def self.prepare_body(config, with_otp) 69 | body = "username=#{CGI.escape(config[:email])}&password=#{CGI.escape(config[:password])}&user_type=RESEARCHER" 70 | 71 | if with_otp 72 | otp_code = ROTP::TOTP.new(config[:otp]).now 73 | body += "&otp_code=#{otp_code}" 74 | end 75 | 76 | body 77 | end 78 | 79 | # Extracts CSRF token from response headers 80 | # @param resp [HTTP::Response] HTTP response object 81 | # @return [String, nil] CSRF token if found, nil otherwise 82 | def self.extract_csrf(resp) 83 | # Vérifier si les headers et set-cookie existent avant d'appeler match 84 | headers = resp&.headers 85 | return nil unless headers 86 | 87 | cookies = headers['set-cookie'] 88 | return nil unless cookies 89 | 90 | match = nil 91 | cookies.each do |cookie| 92 | next if match 93 | 94 | match = cookie.match(%r{csrf-token=(?[\w+/]+)}) 95 | end 96 | 97 | match ? match[:csrf] : nil 98 | end 99 | 100 | # Checks if authentication was successful by following redirects 101 | # @param redirect_to [String] URL to redirect to after login 102 | # @return [Boolean] True if authenticated successfully, false otherwise 103 | def self.check_authentication_success(redirect_to) 104 | resp = HttpClient.get(redirect_to, { follow_location: true }) 105 | return { error: 'Error during follow redirect flow' } unless resp 106 | 107 | success = authenticated?(resp) 108 | { success: success } 109 | end 110 | 111 | def self.authenticated?(resp) 112 | location = resp&.headers&.[]('location') 113 | resp&.body&.include?('Dashboard - Bugcrowd') || location == DASHBOARD_URL 114 | end 115 | 116 | # Validates HTTP response 117 | # @param resp [HTTP::Response] HTTP response to validate 118 | # @param expected_status [Integer] Expected HTTP status code 119 | # @return [Boolean] True if response is valid, false otherwise 120 | def self.valid_response?(resp, expected_status) 121 | !resp.nil? && resp.code == expected_status 122 | end 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /libs/platforms/bugcrowd/programs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'scopes' 4 | 5 | module ScopesExtractor 6 | module Bugcrowd 7 | # Bugcrowd Sync Programs 8 | module Programs 9 | PROGRAMS_ENDPOINT = 'https://bugcrowd.com/engagements.json' 10 | 11 | def self.sync(results, page_id = 1) 12 | url = File.join(PROGRAMS_ENDPOINT, "?page=#{page_id}&category=bug_bounty") 13 | resp = HttpClient.get(url) 14 | return unless resp&.code == 200 15 | 16 | body = Parser.json_parse(resp.body) 17 | return if body['engagements'].empty? 18 | 19 | parse_programs(body['engagements'], results) 20 | sync(results, page_id + 1) 21 | end 22 | 23 | def self.parse_programs(programs, results) 24 | programs.each do |program| 25 | next unless program['accessStatus'] == 'open' 26 | 27 | infos = program_info(program) 28 | scopes = Scopes.sync(program) 29 | 30 | results[program['name']] = infos 31 | results[program['name']]['scopes'] = scopes 32 | end 33 | end 34 | 35 | def self.program_info(program) 36 | slug = program['briefUrl'][1..] 37 | { 38 | slug: slug, 39 | enabled: true, 40 | private: false 41 | } 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /libs/platforms/bugcrowd/scopes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | module Bugcrowd 5 | # Bugcrowd module handles fetching and parsing scope information for Bugcrowd bug bounty programs 6 | module Scopes 7 | # Mapping of Bugcrowd scope types to standardized categories 8 | CATEGORIES = { 9 | web: %w[website api], 10 | mobile: %w[android ios], 11 | other: %w[other], 12 | executable: %w[application], 13 | hardware: %w[hardware], 14 | iot: %w[iot], 15 | network: %w[network], 16 | source_code: %w[code] 17 | }.freeze 18 | 19 | # Constants for URL construction 20 | BASE_URL = 'https://bugcrowd.com' 21 | OUT_OF_SCOPE_MARKERS = ['oos', 'out of scope'].freeze 22 | 23 | # Synchronizes scope information for a Bugcrowd program 24 | # @param program [Hash] Program information hash containing brief_url 25 | # @return [Hash] Hash of in-scope and out-of-scope items categorized 26 | def self.sync(program) 27 | targets = extract_targets(program['briefUrl']) 28 | return { 'in' => {}, 'out' => {} } unless targets 29 | 30 | { 31 | 'in' => parse_scopes(targets), 32 | 'out' => {} # TODO: Implement out-of-scope parsing when available 33 | } 34 | end 35 | 36 | # Parses scope data into categorized formats 37 | # @param targets [Array] Array of target data objects 38 | # @return [Hash] Categorized scope data 39 | def self.parse_scopes(targets) 40 | scopes = {} 41 | 42 | targets.each do |target| 43 | category = find_category(target) 44 | next unless category 45 | 46 | scopes[category] ||= [] 47 | add_scope_to_category(scopes, category, target) 48 | end 49 | 50 | scopes 51 | end 52 | 53 | # Adds a scope to the appropriate category, with normalization for web scopes 54 | # @param scopes [Hash] Scopes hash to add to 55 | # @param category [Symbol] Category to add the scope to 56 | # @param target [Hash] Target information 57 | # @return [void] 58 | def self.add_scope_to_category(scopes, category, target) 59 | if category == :web 60 | Normalizer.run('Bugcrowd', target['name'])&.each { |url| scopes[category] << url } 61 | else 62 | scopes[category] << target['name'] 63 | end 64 | end 65 | 66 | # Finds the standardized category for a target item 67 | # @param target [Hash] Target item information 68 | # @return [Symbol, nil] Standardized category or nil if not found 69 | def self.find_category(target) 70 | category = CATEGORIES.find { |_key, values| values.include?(target['category']) }&.first 71 | Utilities.log_warn("Bugcrowd - Unknown category: #{target}") if category.nil? 72 | 73 | category = :source_code if source_code?(target['name']) 74 | 75 | category 76 | end 77 | 78 | # Determines if a scope is a source code repository 79 | # @param scope [String] Scope value 80 | # @return [Boolean] True if the scope is a GitHub repository 81 | def self.source_code?(scope) 82 | scope&.start_with?('https://github.com/') 83 | end 84 | 85 | # Extracts targets from Bugcrowd program brief URL 86 | # @param brief_url [String] Program brief URL 87 | # @return [Array, nil] Array of targets or nil if extraction fails 88 | def self.extract_targets(brief_url) 89 | url = File.join(BASE_URL, brief_url) 90 | 91 | if brief_url.start_with?('/engagements/') 92 | targets_from_engagements(url) 93 | else 94 | targets_from_groups(url) 95 | end 96 | end 97 | 98 | # Extracts targets from engagement-type programs 99 | # @param url [String] Program URL 100 | # @return [Array, nil] Array of targets or nil if extraction fails 101 | def self.targets_from_engagements(url) 102 | # Fetch and extract changelog ID 103 | changelog_id = fetch_changelog_id(url) 104 | return nil unless changelog_id 105 | 106 | # Fetch targets from changelog 107 | targets = fetch_targets_from_changelog(url, changelog_id) 108 | return nil unless targets 109 | 110 | # Process targets 111 | process_engagement_targets(targets) 112 | end 113 | 114 | # Fetches changelog ID from engagement page 115 | # @param url [String] Program URL 116 | # @return [String, nil] Changelog ID or nil if not found 117 | def self.fetch_changelog_id(url) 118 | response = fetch_with_logging(url, 'engagement page') 119 | return nil unless response 120 | 121 | match = response.body.match(%r{changelog/(?<changelog>[-a-f0-9]+)}) 122 | unless match 123 | Discord.log_warn("Bugcrowd - Failed to extract changelog ID from: #{url}") 124 | return nil 125 | end 126 | 127 | match[:changelog] 128 | end 129 | 130 | # Fetches targets from changelog 131 | # @param url [String] Base program URL 132 | # @param changelog_id [String] Changelog ID 133 | # @return [Array, nil] Array of scope objects or nil if fetch fails 134 | def self.fetch_targets_from_changelog(url, changelog_id) 135 | changelog_url = File.join(url, 'changelog', "#{changelog_id}.json") 136 | response = fetch_with_logging(changelog_url, 'changelog') 137 | return nil unless response 138 | 139 | json = parse_json_with_logging(response.body, changelog_url) 140 | return nil unless json 141 | 142 | json.dig('data', 'scope') 143 | end 144 | 145 | # Processes targets from engagement scopes 146 | # @param scopes [Array] Array of scope objects 147 | # @return [Array] Array of flattened targets 148 | def self.process_engagement_targets(scopes) 149 | targets = [] 150 | 151 | scopes.each do |scope| 152 | # Skip out-of-scope items 153 | next if OUT_OF_SCOPE_MARKERS.any? { |marker| scope['name'].downcase.include?(marker) } 154 | 155 | targets << scope['targets'] 156 | end 157 | 158 | targets.flatten 159 | end 160 | 161 | # Extracts targets from group-type programs 162 | # @param url [String] Program URL 163 | # @return [Array, nil] Array of targets or nil if extraction fails 164 | def self.targets_from_groups(url) 165 | # Fetch target groups 166 | groups = fetch_target_groups(url) 167 | return nil unless groups 168 | 169 | # Process each group and collect targets 170 | targets = [] 171 | groups.each do |group| 172 | # Skip out-of-scope groups 173 | next unless group['in_scope'] 174 | 175 | # Fetch targets for this group 176 | group_targets = fetch_group_targets(group['targets_url']) 177 | targets << group_targets if group_targets 178 | end 179 | 180 | targets.flatten 181 | end 182 | 183 | # Fetches target groups 184 | # @param url [String] Program URL 185 | # @return [Array, nil] Array of group objects or nil if fetch fails 186 | def self.fetch_target_groups(url) 187 | groups_url = File.join(url, 'target_groups') 188 | headers = { 'Accept' => 'application/json' } 189 | response = HttpClient.get(groups_url, { headers: headers }) 190 | 191 | unless valid_response?(response) 192 | Discord.log_warn("Bugcrowd - Failed to fetch target groups: #{groups_url}") 193 | return nil 194 | end 195 | 196 | json = parse_json_with_logging(response.body, groups_url) 197 | return nil unless json 198 | 199 | json['groups'] 200 | end 201 | 202 | # Fetches targets for a specific group 203 | # @param targets_url [String] Targets URL from group data 204 | # @return [Array, nil] Array of targets or nil if fetch fails 205 | def self.fetch_group_targets(targets_url) 206 | full_targets_url = File.join(BASE_URL, targets_url) 207 | response = fetch_with_logging(full_targets_url, 'targets') 208 | return nil unless response 209 | 210 | json = parse_json_with_logging(response.body, full_targets_url) 211 | return nil unless json 212 | 213 | json['targets'] 214 | end 215 | 216 | # Helper method for fetching with error logging 217 | # @param url [String] URL to fetch 218 | # @param resource_type [String] Description of what's being fetched for logging 219 | # @return [HTTP::Response, nil] Response object or nil if request failed 220 | def self.fetch_with_logging(url, resource_type) 221 | retries = 0 222 | max_retries = 2 223 | 224 | loop do 225 | response = HttpClient.get(url) 226 | return response if valid_response?(response) 227 | 228 | retries += 1 229 | if retries <= max_retries 230 | sleep 3 # Sleep for 3 seconds before retrying 231 | else 232 | Discord.log_warn("Bugcrowd - Failed to fetch #{resource_type}: #{url} after #{max_retries} retries") 233 | return nil 234 | end 235 | end 236 | end 237 | 238 | # Helper method for parsing JSON with error logging 239 | # @param body [String] Response body to parse 240 | # @param url [String] URL for logging 241 | # @return [Hash, nil] Parsed JSON or nil if parsing failed 242 | def self.parse_json_with_logging(body, url) 243 | json = Parser.json_parse(body) 244 | unless json 245 | Discord.log_warn("Bugcrowd - Failed to parse JSON from: #{url}") 246 | return nil 247 | end 248 | json 249 | end 250 | 251 | # Validates HTTP response 252 | # @param response [HTTP::Response] HTTP response to validate 253 | # @return [Boolean] True if response is valid, false otherwise 254 | def self.valid_response?(response) 255 | !response.nil? && response.code == 200 256 | end 257 | end 258 | end 259 | end 260 | -------------------------------------------------------------------------------- /libs/platforms/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'dotenv/load' 4 | 5 | module ScopesExtractor 6 | # Config class manages application configuration loaded from environment variables 7 | # for various services and APIs used in the application 8 | class Config 9 | # Loads configuration from environment variables 10 | # @return [Hash] A hash containing application configuration 11 | def self.load 12 | { 13 | yeswehack: yeswehack_config, 14 | intigriti: intigriti_config, 15 | hackerone: hackerone_config, 16 | bugcrowd: bugcrowd_config, 17 | immunefi: immunefi_config, 18 | discord: discord_config, 19 | api: api_config, 20 | sync: sync_config, 21 | history: history_config 22 | } 23 | end 24 | 25 | # Private class methods for configuration segments 26 | class << self 27 | private 28 | 29 | def yeswehack_config 30 | { 31 | enabled: ENV.fetch('YWH_SYNC', false), 32 | email: ENV.fetch('YWH_EMAIL', nil), 33 | password: ENV.fetch('YWH_PWD', nil), 34 | otp: ENV.fetch('YWH_OTP', nil) 35 | } 36 | end 37 | 38 | def intigriti_config 39 | { 40 | enabled: ENV.fetch('INTIGRITI_SYNC', false), 41 | token: ENV.fetch('INTIGRITI_TOKEN', nil) 42 | } 43 | end 44 | 45 | def hackerone_config 46 | { 47 | enabled: ENV.fetch('H1_SYNC', false), 48 | username: ENV.fetch('H1_USERNAME', nil), 49 | token: ENV.fetch('H1_TOKEN', nil) 50 | } 51 | end 52 | 53 | def bugcrowd_config 54 | { 55 | enabled: ENV.fetch('BC_SYNC', false), 56 | email: ENV.fetch('BC_EMAIL', nil), 57 | password: ENV.fetch('BC_PWD', nil), 58 | otp: ENV.fetch('BC_OTP', nil) 59 | } 60 | end 61 | 62 | def immunefi_config 63 | { 64 | enabled: ENV.fetch('IMMUNEFI_SYNC', false) 65 | } 66 | end 67 | 68 | def discord_config 69 | { 70 | message_webhook: ENV.fetch('DISCORD_WEBHOOK', nil), 71 | logs_webhook: ENV.fetch('DISCORD_LOGS_WEBHOOK', nil), 72 | notify_categories: ENV.fetch('NOTIFY_CATEGORIES', 'all'), 73 | headers: { 'Content-Type' => 'application/json' } 74 | } 75 | end 76 | 77 | def api_config 78 | { 79 | enabled: ENV.fetch('API_MODE', false), 80 | key: ENV.fetch('API_KEY', nil) 81 | } 82 | end 83 | 84 | def sync_config 85 | { 86 | auto: ENV.fetch('AUTO_SYNC', false), 87 | delay: ENV.fetch('SYNC_DELAY', 10_800) 88 | } 89 | end 90 | 91 | def history_config 92 | { 93 | retention_days: ENV.fetch('HISTORY_RETENTION_DAYS', 30).to_i 94 | } 95 | end 96 | end 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /libs/platforms/hackerone/programs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'scopes' 4 | 5 | module ScopesExtractor 6 | module Hackerone 7 | # Hackerone Sync Programs 8 | module Programs 9 | PROGRAMS_ENDPOINT = 'https://api.hackerone.com/v1/hackers/programs' 10 | 11 | def self.sync(results, config, page_id = 1) 12 | page_infos = get_programs_infos(page_id, config) 13 | return unless page_infos 14 | 15 | parse_programs(page_infos[:programs], config, results) 16 | sync(results, config, page_id + 1) if page_infos[:next_page] 17 | end 18 | 19 | def self.parse_programs(programs, config, results) 20 | programs.each do |program| 21 | attributes = program['attributes'] 22 | next unless attributes['submission_state'] == 'open' && attributes['offers_bounties'] 23 | 24 | name = attributes['name'] 25 | results[name] = program_info(program) 26 | results[name]['scopes'] = Scopes.sync(program_info(program), config) 27 | end 28 | end 29 | 30 | def self.program_info(program) 31 | { 32 | slug: program['attributes']['handle'], 33 | enabled: true, 34 | private: !program['attributes']['state'] == 'public_mode' 35 | } 36 | end 37 | 38 | def self.get_programs_infos(page_id, config) 39 | url = PROGRAMS_ENDPOINT + "?page%5Bnumber%5D=#{page_id}" 40 | response = HttpClient.get(url, { headers: config[:headers] }) 41 | if response&.code == 429 42 | sleep 65 # Rate limit 43 | programs_infos(page_id) 44 | end 45 | return unless response.code == 200 46 | 47 | json = Parser.json_parse(response.body) 48 | return unless json 49 | 50 | { next_page: json.dig('links', 'next'), programs: json['data'] } 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /libs/platforms/hackerone/scopes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | module Hackerone 5 | # Hackerone Sync Scopes 6 | module Scopes 7 | CATEGORIES = { 8 | web: %w[URL WILDCARD IP_ADDRESS API], 9 | cidr: %w[CIDR], 10 | mobile: %w[GOOGLE_PLAY_APP_ID OTHER_APK APPLE_STORE_APP_ID TESTFLIGHT OTHER_IPA], 11 | other: %w[OTHER AWS_CLOUD_CONFIG], 12 | executable: %w[DOWNLOADABLE_EXECUTABLES WINDOWS_APP_STORE_APP_ID], 13 | hardware: %w[HARDWARE], 14 | ai: %w[AI_MODEL], 15 | source_code: %w[SOURCE_CODE SMART_CONTRACT] 16 | }.freeze 17 | 18 | PROGRAMS_ENDPOINT = 'https://api.hackerone.com/v1/hackers/programs' 19 | 20 | def self.sync(program, config) 21 | url = File.join(PROGRAMS_ENDPOINT, program[:slug]) 22 | response = HttpClient.get(url, { headers: config[:headers] }) 23 | return unless response&.code == 200 24 | 25 | json = Parser.json_parse(response.body) 26 | return unless json 27 | 28 | data = json.dig('relationships', 'structured_scopes', 'data') 29 | return unless data 30 | 31 | { 32 | 'in' => parse_scopes(data), 33 | 'out' => {} # TODO 34 | } 35 | end 36 | 37 | def self.parse_scopes(targets) 38 | scopes = {} 39 | 40 | targets.each do |target| 41 | attributes = target['attributes'] 42 | next unless attributes['eligible_for_bounty'] && attributes['eligible_for_submission'] 43 | 44 | category = find_category(attributes) 45 | next unless category 46 | 47 | scopes[category] ||= [] 48 | add_scope_to_category(scopes, category, attributes) 49 | end 50 | 51 | scopes 52 | end 53 | 54 | # Adds a scope to the appropriate category, with normalization for web scopes 55 | # @param scopes [Hash] Scopes hash to add to 56 | # @param category [Symbol] Category to add the scope to 57 | # @param infos [Hash] Scope information 58 | # @return [void] 59 | def self.add_scope_to_category(scopes, category, infos) 60 | if category == :web 61 | Normalizer.run('Hackerone', infos['asset_identifier'])&.each { |url| scopes[category] << url } 62 | else 63 | scopes[category] << infos['asset_identifier'] 64 | end 65 | end 66 | 67 | # Finds the standardized category for a scope item 68 | # @param infos [Hash] Scope item information 69 | # @return [Symbol, nil] Standardized category or nil if not found 70 | def self.find_category(infos) 71 | category = CATEGORIES.find { |_key, values| values.include?(infos['asset_type']) }&.first 72 | Discord.log_warn("Hackerone - Unknown category: #{infos}") if category.nil? 73 | 74 | category = :source_code if source_code?(infos['asset_identifier']) 75 | 76 | category 77 | end 78 | 79 | # Determines if a scope is a source code repository 80 | # @param scope [String] Scope value 81 | # @return [Boolean] True if the scope is a GitHub repository 82 | def self.source_code?(scope) 83 | scope&.start_with?('https://github.com/') 84 | end 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /libs/platforms/immunefi/programs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'nokogiri' 4 | require 'json' 5 | require_relative 'scopes' 6 | 7 | module ScopesExtractor 8 | module Immunefi 9 | # Programs module handles fetching and parsing bug bounty programs from Immunefi platform 10 | module Programs 11 | # Synchronizes Immunefi programs data 12 | # @param results [Hash] Hash to store the fetched programs data 13 | # @return [void] 14 | def self.sync(results) 15 | html = programs_page 16 | return unless html 17 | 18 | parse_programs(html, results) 19 | end 20 | 21 | # Fetches the Immunefi bug bounty programs page 22 | # @return [String, nil] HTML content of the bug bounty page or nil if request fails 23 | def self.programs_page 24 | response = HttpClient.get('https://immunefi.com/bug-bounty/') 25 | return unless response&.code == 200 26 | 27 | response.body 28 | end 29 | 30 | # Parses HTML content to extract program data 31 | # @param html [String] HTML content to parse 32 | # @param results [Hash] Hash to store the parsed programs data 33 | # @return [void] 34 | def self.parse_programs(html, results) 35 | programs = extract_programs(html) 36 | 37 | programs.each do |program| 38 | sleep(1) # Avoid rate limit 39 | title = program['project'] 40 | 41 | program_info = { slug: program['id'], private: false } 42 | 43 | results[title] = program_info 44 | results[title]['scopes'] = Scopes.sync(program_info) 45 | end 46 | end 47 | 48 | # Extracts programs data from HTML using Nokogiri 49 | # @param html [String] HTML content to parse 50 | # @return [Array] Array of program hashes, empty array if extraction fails 51 | def self.extract_programs(html) 52 | doc = Nokogiri::HTML(html) 53 | next_data = doc.at_css('#__NEXT_DATA__') 54 | return [] unless next_data 55 | 56 | json = Parser.json_parse(next_data.text) 57 | return [] unless json 58 | 59 | # Retrieve program list from "props.pageProps.bounties" 60 | programs = json.dig('props', 'pageProps', 'bounties') 61 | return [] unless programs.is_a?(Array) 62 | 63 | programs 64 | end 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /libs/platforms/immunefi/scopes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'nokogiri' 4 | require 'json' 5 | 6 | module ScopesExtractor 7 | module Immunefi 8 | # Scopes module handles fetching and parsing scope information for Immunefi bug bounty programs 9 | module Scopes 10 | # Mapping of scope types to standardized categories 11 | CATEGORIES = { 12 | web: %w[websites_and_applications], 13 | contracts: %w[smart_contract], 14 | blockchain: %w[blockchain_dlt], 15 | source_code: %w[] 16 | }.freeze 17 | 18 | # Synchronizes scope information for an Immunefi program 19 | # @param program [Hash] Program information hash containing slug and private flag 20 | # @return [Hash] Hash of in-scope and out-of-scope items categorized 21 | def self.sync(program) 22 | scopes = { 'in' => {}, 'out' => {} } 23 | response = HttpClient.get("https://immunefi.com/bug-bounty/#{program[:slug]}/information/") 24 | 25 | json = extract_json(program, response) 26 | return scopes unless json 27 | 28 | bounty = json.dig('props', 'pageProps', 'bounty') 29 | unless bounty 30 | Discord.log_warn("Immunefi - No bounty data found for program #{program[:slug]}") 31 | return scopes 32 | end 33 | 34 | assets = bounty['assets'] 35 | scopes['in'] = parse_scopes(assets) 36 | 37 | scopes 38 | end 39 | 40 | # Extracts JSON data from the HTTP response 41 | # @param program [Hash] Program information hash containing slug and private flag 42 | # @param response [Faraday::Response] HTTP response 43 | # @return [Hash, nil] Parsed JSON data or nil if extraction fails 44 | def self.extract_json(program, response) 45 | unless response&.code == 200 46 | Discord.log_warn("Immunefi - Failed to fetch program #{program[:slug]} - #{response&.code}") 47 | return nil 48 | end 49 | 50 | next_data = extract_next_data(program, response) 51 | return unless next_data 52 | 53 | json = Parser.json_parse(next_data.text) 54 | unless json 55 | Discord.log_warn("Immunefi - JSON parsing failed for program #{program[:slug]}") 56 | return nil 57 | end 58 | 59 | json 60 | end 61 | 62 | # Extracts NEXT_DATA element from the response HTML 63 | # @param program [Hash] Program information hash containing slug and private flag 64 | # @param response [Faraday::Response] HTTP response 65 | # @return [Nokogiri::XML::Element, nil] NEXT_DATA element or nil if not found 66 | def self.extract_next_data(program, response) 67 | html = response.body 68 | doc = Nokogiri::HTML(html) 69 | next_data = doc.at_css('#__NEXT_DATA__') 70 | unless next_data 71 | Discord.log_warn("Immunefi - __NEXT_DATA__ element not found for program #{program[:slug]}") 72 | return nil 73 | end 74 | 75 | next_data 76 | end 77 | 78 | # Parses scope data into categorized formats 79 | # @param data [Array] Array of scope data objects 80 | # @return [Hash] Categorized scope data 81 | def self.parse_scopes(data) 82 | return {} unless data.is_a?(Array) 83 | 84 | scopes = {} 85 | 86 | data.each do |infos| 87 | category = find_category(infos) 88 | next unless category 89 | 90 | scopes[category] ||= [] 91 | scopes[category] << infos['url'] 92 | end 93 | 94 | scopes 95 | end 96 | 97 | # Finds the standardized category for a scope item 98 | # @param infos [Hash] Scope item information 99 | # @return [Symbol, nil] Standardized category or nil if not found 100 | def self.find_category(infos) 101 | category = CATEGORIES.find { |_key, values| values.include?(infos['type']) }&.first 102 | Discord.log_warn("Immunefi - Unknown category: #{infos}") if category.nil? 103 | 104 | # Special handling for GitHub repositories 105 | category = :source_code if category == :web && infos['url']&.start_with?('https://github.com/') 106 | 107 | category 108 | end 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /libs/platforms/intigriti/programs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'scopes' 4 | 5 | module ScopesExtractor 6 | module Intigriti 7 | # Intigrit Sync Programs 8 | module Programs 9 | PROGRAMS_ENDPOINT = 'https://api.intigriti.com/external/researcher/v1/programs?limit=500&statusId=3' 10 | 11 | def self.sync(results, config) 12 | response = HttpClient.get(PROGRAMS_ENDPOINT, { headers: config[:headers] }) 13 | return unless response&.code == 200 14 | 15 | data = Parser.json_parse(response.body) 16 | return unless data 17 | 18 | parse_programs(data['records'], config, results) 19 | end 20 | 21 | def self.parse_programs(programs, config, results) 22 | programs&.each do |program| 23 | next if skip_program?(program) 24 | 25 | sleep(0.3) # Avoid rate limit 26 | name = program['name'] 27 | 28 | results[name] = program_info(program) 29 | results[name][:scopes] = Scopes.sync(program, config[:headers]) 30 | end 31 | end 32 | 33 | def self.skip_program?(program) 34 | !program['maxBounty']['value'].positive? 35 | end 36 | 37 | def self.program_info(program) 38 | { 39 | slug: program['handle'], 40 | enabled: true, 41 | private: program.dig('confidentialityLevel', 'id') != 4 # == public 42 | } 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /libs/platforms/intigriti/scopes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | module Intigriti 5 | # Intigrit Sync Scopes 6 | module Scopes 7 | CATEGORIES = { 8 | web: [1, 7], 9 | mobile: [2, 3], 10 | cidr: [4], 11 | device: [5], 12 | other: [6] 13 | }.freeze 14 | 15 | PROGRAMS_ENDPOINT = 'https://api.intigriti.com/external/researcher/v1/programs' 16 | 17 | DENY = [ 18 | '.ripe.net' 19 | ].freeze 20 | 21 | def self.sync(program, headers) 22 | scopes = { 'in' => {}, 'out' => {} } 23 | 24 | response = HttpClient.get("#{PROGRAMS_ENDPOINT}/#{program['id']}", { headers: headers }) 25 | 26 | json = extract_json(program, response) 27 | return scopes unless json 28 | 29 | parse_scopes(json, scopes) 30 | 31 | scopes 32 | end 33 | 34 | # Extracts JSON data from the HTTP response 35 | # @param program [Hash] Program information hash containing id 36 | # @param response [Faraday::Response] HTTP response 37 | # @return [Hash, nil] Parsed JSON data or nil if extraction fails 38 | def self.extract_json(program, response) 39 | unless response&.code == 200 40 | Discord.log_warn("Intigriti - Failed to fetch program #{program['name']} - #{response&.code}") 41 | return nil 42 | end 43 | 44 | json = Parser.json_parse(response.body) 45 | unless json 46 | Discord.log_warn("Intigriti - Failed to parse JSON for program #{program['handle']}") 47 | return nil 48 | end 49 | 50 | json = json.dig('domains', 'content') 51 | unless json 52 | Discord.log_warn("Intigriti - No content for program #{program['handle']}") 53 | return nil 54 | end 55 | 56 | json 57 | end 58 | 59 | def self.parse_scopes(json, scopes) 60 | return unless json.is_a?(Array) 61 | 62 | json.each do |scope| 63 | next unless valid_scope?(scope) 64 | 65 | category = find_category(scope) 66 | type = determine_scope_type(scope) 67 | 68 | scopes[type][category] ||= [] 69 | 70 | add_scope(scopes, type, category, scope) 71 | end 72 | end 73 | 74 | def self.valid_scope?(scope) 75 | return false if scope.dig('tier', 'value') == 'No Bounty' 76 | 77 | category = find_category(scope) 78 | return false unless category 79 | return false if DENY.any? { |deny| scope['endpoint'].include?(deny) } 80 | 81 | true 82 | end 83 | 84 | def self.determine_scope_type(scope) 85 | scope.dig('tier', 'value') == 'Out Of Scope' ? 'out' : 'in' 86 | end 87 | 88 | def self.add_scope(scopes, type, category, scope) 89 | if category == :web && type == 'in' 90 | Normalizer.run('Intigriti', scope['endpoint'])&.each { |url| scopes[type][category] << url } 91 | else 92 | scopes[type][category] << scope['endpoint'].downcase 93 | end 94 | end 95 | 96 | def self.find_category(scope) 97 | category = CATEGORIES.find { |_key, values| values.include?(scope.dig('type', 'id')) }&.first 98 | Utilities.log_warn("Intigriti - Inexistent categories : #{scope}") if category.nil? 99 | 100 | category 101 | end 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /libs/platforms/yeswehack/auth.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # YesWeHack module handles all interactions with the YesWeHack bug bounty platform 5 | module YesWeHack 6 | # API endpoint URLs 7 | LOGIN_URL = 'https://api.yeswehack.com/login' 8 | OTP_LOGIN_URL = 'https://api.yeswehack.com/account/totp' 9 | 10 | # Authenticates with YesWeHack using email, password and TOTP 11 | # @param config [Hash] Configuration containing credentials 12 | # @return [String, nil] JWT token if authentication is successful, nil otherwise 13 | def self.authenticate(config) 14 | response = extract_totp_token(config) 15 | return response if response[:error] 16 | 17 | extract_jwt(response[:totp], config) 18 | end 19 | 20 | # Extracts a TOTP token by authenticating with email and password 21 | # @param config [Hash] Configuration containing email and password 22 | # @return [String, nil] TOTP token if first authentication step is successful, nil otherwise 23 | def self.extract_totp_token(config) 24 | body = { email: config[:email], password: config[:password] }.to_json 25 | 26 | response = HttpClient.post(LOGIN_URL, { body: body }) 27 | return { error: 'Invalid login or password' } unless response&.code == 200 28 | 29 | json = Parser.json_parse(response.body) 30 | return { error: 'Invalid response' } unless json 31 | 32 | { totp: json['totp_token'] } 33 | end 34 | 35 | # Extracts a JWT token by authenticating with a TOTP token and OTP code 36 | # @param totp_token [String] TOTP token from the first authentication step 37 | # @param config [Hash] Configuration containing the OTP secret 38 | # @return [String, nil] JWT token if second authentication step is successful, nil otherwise 39 | def self.extract_jwt(totp_token, config) 40 | otp_code = ROTP::TOTP.new(config[:otp]).now 41 | body = { token: totp_token, code: otp_code }.to_json 42 | 43 | response = HttpClient.post(OTP_LOGIN_URL, { body: body }) 44 | return { error: 'Invalid OTP' } unless response.code == 200 45 | 46 | json = Parser.json_parse(response.body) 47 | return { error: 'Invalid response' } unless json 48 | 49 | { jwt: json['token'] } 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /libs/platforms/yeswehack/programs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative 'scopes' 4 | 5 | module ScopesExtractor 6 | module YesWeHack 7 | # Programs module handles fetching and parsing YesWeHack bug bounty programs 8 | module Programs 9 | PROGRAMS_URL = 'https://api.yeswehack.com/programs' 10 | 11 | # Synchronizes YesWeHack programs data, handling pagination 12 | # @param results [Hash] Hash to store the fetched programs data 13 | # @param config [Hash] Configuration hash with authentication headers 14 | # @param page_id [Integer] Page number for pagination, defaults to 1 15 | # @return [void] 16 | def self.sync(results, config, page_id = 1) 17 | page_infos = get_page_infos(page_id, config) 18 | return unless page_infos 19 | 20 | parse_programs(page_infos[:programs], results, config) 21 | sync(results, config, page_id + 1) unless page_id == page_infos[:nb_pages] 22 | end 23 | 24 | # Gets program information for a specific page 25 | # @param page_id [Integer] Page number to fetch 26 | # @param config [Hash] Configuration hash with authentication headers 27 | # @return [Hash, nil] Hash containing page count and programs, or nil on failure 28 | def self.get_page_infos(page_id, config) 29 | response = HttpClient.get("#{PROGRAMS_URL}?page=#{page_id}", { headers: config[:headers] }) 30 | return unless response&.code == 200 31 | 32 | json = Parser.json_parse(response.body) 33 | return unless json 34 | 35 | { nb_pages: json.dig('pagination', 'nb_pages'), programs: json['items'] } 36 | end 37 | 38 | # Parses program data and adds it to the results hash 39 | # @param programs [Array] Array of program data objects 40 | # @param results [Hash] Hash to store the parsed program data 41 | # @param config [Hash] Configuration hash with authentication headers 42 | # @return [void] 43 | def self.parse_programs(programs, results, config) 44 | programs.each do |program| 45 | next if program['disabled'] || program['vdp'] 46 | 47 | title = program['title'] 48 | 49 | program_info = { slug: program['slug'], private: !program['public'] } 50 | 51 | results[title] = program_info 52 | results[title]['scopes'] = Scopes.sync(program_info, config) 53 | end 54 | end 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /libs/platforms/yeswehack/scopes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | module YesWeHack 5 | # Scopes module handles fetching and parsing scope information for YesWeHack bug bounty programs 6 | module Scopes 7 | # Mapping of YesWeHack scope types to standardized categories 8 | CATEGORIES = { 9 | web: %w[web-application api ip-address], 10 | mobile: %w[mobile-application mobile-application-android mobile-application-ios], 11 | source_code: %w[], 12 | other: %w[other], 13 | executable: %w[application] 14 | }.freeze 15 | 16 | BASE_SCOPE_URL = 'https://api.yeswehack.com/programs' 17 | 18 | # Synchronizes scope information for a YesWeHack program 19 | # @param program [Hash] Program information hash containing slug 20 | # @param config [Hash] Configuration hash with authentication headers 21 | # @return [Hash] Hash of in-scope and out-of-scope items categorized 22 | def self.sync(program, config) 23 | scopes = { 'in' => {}, 'out' => {} } 24 | response = HttpClient.get("#{BASE_SCOPE_URL}/#{program[:slug]}", { headers: config[:headers] }) 25 | 26 | json = extract_json(program, response) 27 | return scopes unless json 28 | 29 | scopes['in'] = parse_scopes(json['scopes'], true) 30 | scopes['out'] = parse_scopes(json['out_of_scope'], false) 31 | 32 | scopes 33 | end 34 | 35 | # Extracts JSON data from the HTTP response 36 | # @param program [Hash] Program information hash containing slug and private flag 37 | # @param response [Faraday::Response] HTTP response 38 | # @return [Hash, nil] Parsed JSON data or nil if extraction fails 39 | def self.extract_json(program, response) 40 | unless response&.code == 200 41 | Discord.log_warn("YesWeHack - Failed to fetch program #{program[:slug]} - #{response&.code}") 42 | return nil 43 | end 44 | 45 | json = Parser.json_parse(response.body) 46 | unless json 47 | Discord.log_warn("YesWeHack - Failed to parse JSON for program #{program[:slug]}") 48 | return nil 49 | end 50 | 51 | json 52 | end 53 | 54 | # Parses scope data into categorized formats 55 | # @param data [Array] Array of scope data objects 56 | # @param in_scope [Boolean] Whether this is in-scope (true) or out-of-scope (false) data 57 | # @return [Hash] Categorized scope data 58 | def self.parse_scopes(data, in_scope) 59 | return {} unless data.is_a?(Array) 60 | 61 | scopes = {} 62 | 63 | data.each do |infos| 64 | category = find_category(infos, in_scope) 65 | next unless category 66 | 67 | scopes[category] ||= [] 68 | add_scope_to_category(scopes, category, infos) 69 | end 70 | 71 | scopes 72 | end 73 | 74 | # Adds a scope to the appropriate category, with normalization for web scopes 75 | # @param scopes [Hash] Scopes hash to add to 76 | # @param category [Symbol] Category to add the scope to 77 | # @param infos [Hash] Scope information 78 | # @return [void] 79 | def self.add_scope_to_category(scopes, category, infos) 80 | if category == :web 81 | Normalizer.run('YesWeHack', infos['scope'])&.each { |url| scopes[category] << url } 82 | else 83 | scopes[category] << infos['scope'] 84 | end 85 | end 86 | 87 | # Finds the standardized category for a scope item 88 | # @param infos [Hash] Scope item information 89 | # @param in_scope [Boolean] Whether this is in-scope data (for warning purposes) 90 | # @return [Symbol, nil] Standardized category or nil if not found 91 | def self.find_category(infos, in_scope) 92 | category = CATEGORIES.find { |_key, values| values.include?(infos['scope_type']) }&.first 93 | Discord.log_warn("YesWeHack - Unknown category: #{infos}") if category.nil? && in_scope 94 | 95 | category = :source_code if source_code?(infos['scope']) 96 | 97 | category 98 | end 99 | 100 | # Determines if a scope is a source code repository 101 | # @param scope [String] Scope value 102 | # @return [Boolean] True if the scope is a GitHub repository 103 | def self.source_code?(scope) 104 | scope&.start_with?('https://github.com/') 105 | end 106 | end 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /libs/scopes_extractor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'json' 4 | require 'rotp' 5 | require 'logger' 6 | require 'base64' 7 | 8 | Dir[File.join(__dir__, 'platforms', '**', '*.rb')].sort.each { |file| require file } 9 | Dir[File.join(__dir__, 'utilities', '**', '*.rb')].sort.each { |file| require file } 10 | Dir[File.join(__dir__, 'db', '*.rb')].sort.each { |file| require file } 11 | 12 | module ScopesExtractor 13 | # The Extract class manages the process of extracting and comparing bug bounty program scopes 14 | # from multiple platforms, handling notifications when changes are detected. 15 | class Extract 16 | # List of supported bug bounty platforms 17 | PLATFORMS = %w[Immunefi YesWeHack Intigriti Hackerone Bugcrowd].freeze 18 | 19 | attr_accessor :config, :results 20 | 21 | # Initialize the extractor with configuration and empty results 22 | # @return [Extract] A new instance of the Extract class 23 | def initialize 24 | @config = Config.load 25 | @results = {} 26 | PLATFORMS.each { |platform| @results[platform] = {} } 27 | end 28 | 29 | # Run the extraction process for all platforms and handle notifications 30 | # @return [void] 31 | def run 32 | sync_platforms 33 | 34 | return unless api_mode? 35 | 36 | require 'webrick' 37 | 38 | server = WEBrick::HTTPServer.new(Port: 4567) 39 | server.mount_proc '/' do |req, res| 40 | api_response(req, res) 41 | end 42 | 43 | trap('INT') { server.shutdown } 44 | server.start 45 | end 46 | 47 | # Gets recent changes from the history 48 | # @param hours [Integer] Number of hours to look back (default: 48) 49 | # @param filters [Hash] Optional filters for the changes (platform, program, change_type) 50 | # @return [Array] Array of recent changes matching the criteria 51 | def get_recent_changes(hours = 48, filters = {}) 52 | DB.get_recent_changes(hours, filters) 53 | end 54 | 55 | private 56 | 57 | # Processes an API request by verifying the API key in the header and returning the appropriate data in JSON. 58 | # 59 | # @param req [WEBrick::HTTPRequest] The incoming HTTP request object. 60 | # @param res [WEBrick::HTTPResponse] The HTTP response object that will be returned. 61 | # @return [void] 62 | def api_response(req, res) 63 | api_key = req.header['x-api-key']&.first 64 | res.content_type = 'application/json' 65 | 66 | return unauthorized_response(res) unless valid_api_key?(api_key) 67 | 68 | path = req.path 69 | query = req.query || {} 70 | 71 | if path.start_with?('/changes') 72 | handle_changes_request(res, query) 73 | else 74 | handle_default_request(res) 75 | end 76 | end 77 | 78 | # Validates if the provided API key matches the configured key. 79 | # 80 | # @param api_key [String, nil] The API key from the request header. 81 | # @return [Boolean] True if the API key is valid, false otherwise. 82 | def valid_api_key?(api_key) 83 | api_key == config.dig(:api, :key) 84 | end 85 | 86 | # Sends an unauthorized response with 401 status code. 87 | # 88 | # @param res [WEBrick::HTTPResponse] The HTTP response object to be modified. 89 | # @return [void] 90 | def unauthorized_response(res) 91 | res.status = 401 92 | res.body = { error: 'Unauthorized' }.to_json 93 | end 94 | 95 | # Handles requests to the /changes endpoint, applying appropriate filters. 96 | # 97 | # @param res [WEBrick::HTTPResponse] The HTTP response object to be modified. 98 | # @param query [Hash] The query parameters from the request. 99 | # @return [void] 100 | def handle_changes_request(res, query) 101 | hours = (query['hours'] || 48).to_i 102 | filters = extract_filters(query) 103 | res.body = get_recent_changes(hours, filters).to_json 104 | end 105 | 106 | # Extracts relevant filters from the query parameters. 107 | # 108 | # @param query [Hash] The query parameters from the request. 109 | # @return [Hash] A hash containing only the non-nil filter values. 110 | def extract_filters(query) 111 | { 112 | platform: query['platform'], 113 | change_type: query['type'], 114 | program: query['program'], 115 | category: query['category'] 116 | }.compact 117 | end 118 | 119 | # Handles the default API request by returning the current state. 120 | # 121 | # @param res [WEBrick::HTTPResponse] The HTTP response object to be modified. 122 | # @return [void] 123 | def handle_default_request(res) 124 | res.body = DB.load.to_json 125 | end 126 | 127 | # Synchronizes the bug bounty platforms. 128 | # 129 | # If auto-sync is enabled in the configuration, this method spawns a new thread that 130 | # repeatedly performs synchronization with a configurable delay. 131 | # Otherwise, it performs a one-time synchronization. 132 | # 133 | # @return [Thread, void] Returns a Thread object if auto-sync is enabled, or nil if running synchronously. 134 | def sync_platforms 135 | Utilities.log_warn("AutoSync Status : #{auto_sync?}") 136 | if auto_sync? 137 | Thread.new do 138 | loop do 139 | perform_sync 140 | delay = config.dig(:sync, :delay)&.to_i 141 | Utilities.log_info("Sleep #{delay}") 142 | sleep(delay) 143 | rescue StandardError => e 144 | Discord.log_error("Error during sync_platform : #{e}") 145 | sleep(delay) 146 | end 147 | end 148 | else 149 | api_mode? ? Thread.new { perform_sync } : perform_sync 150 | end 151 | end 152 | 153 | # Performs the synchronization of platforms. 154 | # 155 | # This method loads the current data from the database, runs the platform-specific sync methods, 156 | # compares the newly fetched data with the existing data to trigger notifications, and finally saves 157 | # the updated results back to the database. 158 | # 159 | # @return [void] 160 | def perform_sync 161 | Utilities.log_info('Start synchronisation') 162 | current_data = DB.load 163 | 164 | yeswehack_sync 165 | immunefi_sync 166 | intigriti_sync 167 | hackerone_sync 168 | bugcrowd_sync 169 | 170 | Utilities::ScopeComparator.compare_and_notify(current_data, results, PLATFORMS) unless current_data.empty? 171 | 172 | DB.save(results) 173 | Utilities.log_info('Synchronisation Finished') 174 | end 175 | 176 | # Determines whether API mode is enabled in the configuration. 177 | # 178 | # @return [Boolean] Returns true if API mode is enabled, false otherwise. 179 | def api_mode? 180 | config.dig(:api, :enabled)&.downcase == 'true' 181 | end 182 | 183 | # Determines whether auto synchronization is enabled in the configuration. 184 | # 185 | # @return [Boolean] Returns true if auto-sync is enabled, false otherwise. 186 | def auto_sync? 187 | config.dig(:sync, :auto)&.downcase == 'true' 188 | end 189 | 190 | # Helper method to handle authentication retries 191 | # @param platform [String] Platform name for logging 192 | # @param max_retries [Integer] Maximum number of retry attempts 193 | # @param retry_delay [Integer] Delay in seconds between retry attempts 194 | # @yield [Block] Block executing the authentication and returning a hash with :error or :success 195 | # @return [Hash, nil] Authentication result or nil if all attempts fail 196 | def with_authentication_retry(platform, max_retries: 2, retry_delay: 30) 197 | retries = 0 198 | 199 | loop do 200 | result = yield 201 | 202 | return result unless result[:error] || (result.key?(:success) && !result[:success]) 203 | 204 | error_msg = result[:error] || 'Unknown error' 205 | Discord.log_warn("#{platform} - Authentication Failed with error: #{error_msg}") 206 | 207 | retries += 1 208 | if retries >= max_retries 209 | Discord.log_warn("#{platform} - Max retries (#{max_retries}) reached. Giving up.") 210 | return nil 211 | end 212 | 213 | Discord.log_info("#{platform} - Retrying in #{retry_delay} seconds... (Attempt #{retries}/#{max_retries})") 214 | sleep(retry_delay) 215 | end 216 | end 217 | 218 | # Checks if YesWeHack is configured with required credentials 219 | # @return [Boolean] True if YesWeHack is configured, false otherwise 220 | def yeswehack_configured? 221 | return false if config.dig(:yeswehack, :enabled) == 'false' 222 | 223 | !!(config.dig(:yeswehack, :email) && config.dig(:yeswehack, :password) && config.dig(:yeswehack, :otp)) 224 | end 225 | 226 | # Syncs data from YesWeHack platform 227 | # @return [void] 228 | def yeswehack_sync 229 | return unless yeswehack_configured? 230 | 231 | auth_result = with_authentication_retry('YesWeHack') do 232 | YesWeHack.authenticate(config[:yeswehack]) 233 | end 234 | return unless auth_result 235 | 236 | config[:yeswehack][:headers] = { 237 | 'Content-Type' => 'application/json', 238 | Authorization: "Bearer #{auth_result[:jwt]}" 239 | } 240 | YesWeHack::Programs.sync(results['YesWeHack'], config[:yeswehack]) 241 | end 242 | 243 | # Checks if Immunefi is configured 244 | # @return [Boolean] True if Immunefi is configured, false otherwise 245 | def immunefi_configured? 246 | config.dig(:immunefi, :enabled) != 'false' 247 | end 248 | 249 | # Syncs data from Immunefi platform 250 | # @return [void] 251 | def immunefi_sync 252 | return unless immunefi_configured? 253 | 254 | Immunefi::Programs.sync(results['Immunefi']) 255 | end 256 | 257 | # Checks if Intigriti is configured with required credentials 258 | # @return [Boolean] True if YesWeHack is configured, false otherwise 259 | def intigriti_configured? 260 | return false if config.dig(:intigriti, :enabled) == 'false' 261 | 262 | config.dig(:intigriti, :token) 263 | end 264 | 265 | # Syncs data from Intigriti platform 266 | # @return [void] 267 | def intigriti_sync 268 | return unless intigriti_configured? 269 | 270 | config[:intigriti][:headers] = { Authorization: "Bearer #{config.dig(:intigriti, :token)}" } 271 | Intigriti::Programs.sync(results['Intigriti'], config[:intigriti]) 272 | end 273 | 274 | # Checks if Hackerone is configured with required credentials 275 | # @return [Boolean] True if Hackerone is configured, false otherwise 276 | def hackerone_configured? 277 | return false if config.dig(:hackerone, :enabled) == 'false' 278 | 279 | config.dig(:hackerone, :username) && config.dig(:hackerone, :token) 280 | end 281 | 282 | # Syncs data from Hackerone platform 283 | # @return [void] 284 | def hackerone_sync 285 | return unless hackerone_configured? 286 | 287 | basic = Base64.urlsafe_encode64("#{config[:hackerone][:username]}:#{config[:hackerone][:token]}") 288 | config[:hackerone][:headers] = { Authorization: "Basic #{basic}" } 289 | 290 | Hackerone::Programs.sync(results['Hackerone'], config[:hackerone]) 291 | end 292 | 293 | # Checks if Bugcrowd is configured with required credentials 294 | # @return [Boolean] True if Hackerone is configured, false otherwise 295 | def bugcrowd_configured? 296 | return false if config.dig(:bugcrowd, :enabled) == 'false' 297 | 298 | config.dig(:bugcrowd, :email) && config.dig(:bugcrowd, :password) && config.dig(:bugcrowd, :otp) 299 | end 300 | 301 | # Syncs data from Bugcrowd platform 302 | # @return [void] 303 | def bugcrowd_sync 304 | return unless bugcrowd_configured? 305 | 306 | auth_result = with_authentication_retry('Bugcrowd') do 307 | Bugcrowd.authenticate(config[:bugcrowd]) 308 | end 309 | return unless auth_result 310 | 311 | Bugcrowd::Programs.sync(results['Bugcrowd']) 312 | end 313 | end 314 | end 315 | -------------------------------------------------------------------------------- /libs/utilities/http_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'typhoeus' 4 | 5 | module ScopesExtractor 6 | # HttpClient module provides a simplified interface for making HTTP requests 7 | # with cookie support and consistent error handling 8 | module HttpClient 9 | Typhoeus::Config.user_agent = 'curl/8.7.1' 10 | @cookie_jar = File.join(__dir__, '../db/cookies.txt') 11 | 12 | # Clears the cookie jar file by either truncating if it exists or creating a new empty file 13 | # @return [Boolean] True if cookie jar was successfully cleared 14 | def self.clear_cookie_jar 15 | File.truncate(@cookie_jar, 0) if File.exist?(@cookie_jar) 16 | true 17 | rescue StandardError => e 18 | Discord.log_warn("Error clearing cookie jar: #{e.message}") 19 | false 20 | end 21 | 22 | # Common request options used for both GET and POST requests 23 | # @param method [Symbol] HTTP method (:get or :post) 24 | # @param options [Hash] Request-specific options 25 | # @return [Hash] Combined request options 26 | def self.build_request_options(method, options = {}) 27 | { 28 | method: method, 29 | headers: options[:headers] || {}, 30 | followlocation: options[:follow_location] || false, 31 | timeout: 30, # Default timeout 32 | cookiefile: @cookie_jar, 33 | cookiejar: @cookie_jar, 34 | body: options[:body] # Will be nil for GET requests 35 | }.compact 36 | end 37 | 38 | # Performs an HTTP request 39 | # @param method [Symbol] HTTP method to use 40 | # @param url [String] The URL to request 41 | # @param options [Hash] Request options including headers and body 42 | # @return [Typhoeus::Response, nil] Response object if successful 43 | def self.request(method, url, options = {}) 44 | request_options = build_request_options(method, options) 45 | Typhoeus::Request.new(url, request_options).run 46 | rescue StandardError => e 47 | Discord.log_warn("HTTP error when requesting URL '#{url}': #{e.message}") 48 | nil 49 | end 50 | 51 | # Performs an HTTP GET request 52 | # @param url [String] The URL to request 53 | # @param options [Hash] Request options including headers 54 | # @return [Typhoeus::Response, nil] Response object if successful 55 | def self.get(url, options = {}) 56 | request(:get, url, options) 57 | end 58 | 59 | # Performs an HTTP POST request 60 | # @param url [String] The URL to request 61 | # @param options [Hash] Request options including body and headers 62 | # @return [Typhoeus::Response, nil] Response object if successful 63 | def self.post(url, options = {}) 64 | request(:post, url, options) 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /libs/utilities/logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'logger' 4 | require 'colorize' 5 | 6 | module ScopesExtractor 7 | # Provides helper methods to be used in all the different classes 8 | module Utilities 9 | # Creates a singleton logger 10 | def self.logger 11 | return @logger if @logger 12 | 13 | @logger = Logger.new($stdout) 14 | @logger.formatter = proc do |severity, datetime, _, msg| 15 | date_format = datetime.strftime('%Y-%m-%d %H:%M:%S') 16 | "[#{date_format}] #{severity} #{msg}\n" 17 | end 18 | 19 | @logger 20 | end 21 | 22 | def self.log_warn(message) 23 | logger.warn(message.yellow) 24 | end 25 | 26 | def self.log_info(message) 27 | logger.info(message.blue) 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /libs/utilities/normalizer/bugcrowd.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # Normalizer module for standardizing domain formats 5 | module Normalizer 6 | # Bugcrowd module provides specialized normalization functions for Bugcrowd platform scopes 7 | module Bugcrowd 8 | # Normalize a domain scope string into standardized format(s) 9 | # @param value [String] The raw domain scope string 10 | # @return [Array<String>] List of normalized domain scopes 11 | def self.normalization(value) 12 | if value.include?(' - ') 13 | [normalize_with_dash(value)] 14 | else 15 | [value] 16 | end 17 | end 18 | 19 | # Extracts the domain part before a dash and description 20 | # @param value [String] The raw domain scope string with dash 21 | # @return [String] The normalized domain without description 22 | def self.normalize_with_dash(value) 23 | value.split(' - ').first.strip 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /libs/utilities/normalizer/hackerone.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # Normalizer module for standardizing domain formats 5 | module Normalizer 6 | # Hackerone module provides specialized normalization functions for Hackerone platform scopes 7 | module Hackerone 8 | # Normalize a domain scope string into standardized format(s) 9 | # @param value [String] The raw domain scope string 10 | # @return [Array<String>] List of normalized domain scopes 11 | def self.normalization(value) 12 | value = value.sub('.*', '.com') 13 | .sub(/\.\(TLD\)/i, '.com') 14 | 15 | if value.include?(',') 16 | value.split(',') 17 | else 18 | [value] 19 | end 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /libs/utilities/normalizer/intigriti.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # Normalizer module for standardizing domain formats 5 | module Normalizer 6 | # Intigriti module provides specialized normalization functions for Intigriti platform scopes 7 | module Intigriti 8 | # Normalize a domain scope string into standardized format(s) 9 | # @param value [String] The raw domain scope string 10 | # @return [Array<String>] List of normalized domain scopes 11 | def self.normalization(value) 12 | value = value.sub(/^\*(\s\.|\.?\s)/, '*.') 13 | .sub('.*', '.com') 14 | .sub(/\.<TLD>/i, '.com') 15 | 16 | if value.include?(' / ') 17 | normalize_with_slash(value) 18 | else 19 | [value] 20 | end 21 | end 22 | 23 | # Extracts the domain when a slash ' / ' is present 24 | # @param value [String] The raw domain scope string with slash 25 | # @return [String] The normalized domain without description 26 | def self.normalize_with_slash(value) 27 | value.split(' / ') 28 | end 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /libs/utilities/normalizer/normalizer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # Normalizer module provides methods to standardize and validate scope formats 5 | # across different bug bounty platforms 6 | module Normalizer 7 | # Regular expression for validating IPv4 addresses 8 | IP_REGEX = /\A((?:\d{1,3}\.){3}(?:\d{1,3})\Z)/.freeze 9 | 10 | # Runs normalization of a scope value based on the platform 11 | # @param platform [String] The platform name (e.g., 'YesWeHack') 12 | # @param value [String] The scope value to normalize 13 | # @return [Array<String>] List of normalized scopes 14 | def self.run(platform, value) 15 | scope = global_normalization(value) 16 | 17 | normalized_scopes = case platform 18 | when 'YesWeHack' 19 | YesWeHack.normalization(scope) 20 | when 'Intigriti' 21 | Intigriti.normalization(scope) 22 | when 'Hackerone' 23 | Hackerone.normalization(scope) 24 | when 'Bugcrowd' 25 | Bugcrowd.normalization(scope) 26 | else 27 | [] 28 | end 29 | 30 | normalized_scopes.uniq! 31 | normalized_scopes.select do |s| 32 | Normalizer.valid?(s) || false 33 | end 34 | end 35 | 36 | # Validates if a scope value is a valid IP address or URI 37 | # @param value [String] The scope value to validate 38 | # @return [Boolean] True if the value is valid, false otherwise 39 | def self.valid?(value) 40 | value.match?(IP_REGEX) ? Parser.valid_ip?(value) : Parser.valid_uri?(value) 41 | end 42 | 43 | # Performs global normalization of a scope value 44 | # @param value [String] The scope value to normalize 45 | # @return [String] The normalized scope value 46 | def self.global_normalization(value) 47 | value = value.strip 48 | value = global_end_strip(value) 49 | 50 | # Remove protocol (http:// or https://) if string matches the pattern 51 | value = value.sub(%r{https?://}, '') if value.match?(%r{https?://\*\.}) 52 | 53 | # Add "*" at the beginning if the string starts with a dot 54 | value = "*#{value}" if value.start_with?('.') 55 | 56 | # Return the lowercase string 57 | value.downcase 58 | end 59 | 60 | # Removes special characters from the end of a scope value 61 | # @param value [String] The scope value to process 62 | # @return [String] The processed scope value 63 | def self.global_end_strip(value) 64 | # Remove last two characters if the string ends with "/*" 65 | value = value[0..-2] if value.end_with?('/*') 66 | 67 | # Remove last character if the string ends with "/" and starts with "*." 68 | value = value[0..-2] if value.end_with?('/') && value.start_with?('*.') 69 | 70 | # Remove first character if the string starts with "*" but not "*." 71 | value[1..] if value.start_with?('*') && !value.start_with?('*.') 72 | value 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /libs/utilities/normalizer/yeswehack.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | # Normalizer module for standardizing domain formats 5 | module Normalizer 6 | # YesWeHack module provides specialized normalization functions for YesWeHack platform scopes 7 | module YesWeHack 8 | # Patterns indicating special cases that should not be normalized 9 | EXCLUDED_PATTERNS = [ 10 | 'endpoints on our sites', 11 | 'special scenarios', 12 | 'core services', 13 | 'see program description', 14 | 'see description' 15 | ].freeze 16 | 17 | # Regex patterns for matching specific domain formats 18 | MULTI_TLDS = %r{(?<prefix>https?://|wss?://|\*\.)?(?<middle>[\w.-]+\.)\((?<tlds>[a-z.|]+)}.freeze 19 | 20 | # Normalize a domain scope string into standardized format(s) 21 | # @param value [String] The raw domain scope string 22 | # @return [Array<String>] List of normalized domain scopes 23 | def self.normalization(value) 24 | return [] if excluded_pattern?(value) 25 | 26 | normalized_scopes = [] 27 | 28 | # Process domains with multiple TLDs in parentheses 29 | if (match = value.match(MULTI_TLDS)) 30 | normalized_scopes.concat(normalize_with_tlds(match)) 31 | end 32 | 33 | normalized_scopes << value if normalized_scopes.empty? 34 | 35 | normalized_scopes 36 | end 37 | 38 | # Check if value contains any excluded patterns 39 | # @param value [String] The scope value to check 40 | # @return [Boolean] True if an excluded pattern is found 41 | def self.excluded_pattern?(value) 42 | EXCLUDED_PATTERNS.any? { |pattern| value.include?(pattern) } 43 | end 44 | 45 | # Normalize domains with multiple TLDs specified in parentheses 46 | # @param match [MatchData] Regex match data from MULTI_TLDS 47 | # @return [Array<String>] List of normalized domains 48 | def self.normalize_with_tlds(match) 49 | match[:tlds].split('|').map { |tld| "#{match[:prefix]}#{match[:middle]}#{tld}" } 50 | end 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /libs/utilities/notifier.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../platforms/config' 4 | 5 | module ScopesExtractor 6 | # Discord module handles sending notification messages to Discord webhooks 7 | # for various events and system logs 8 | module Discord 9 | # Loads Discord configuration from the global config 10 | # @return [Hash] Discord configuration 11 | def self.config 12 | @config ||= ScopesExtractor::Config.load[:discord] 13 | end 14 | 15 | # Sends a warning log message to Discord 16 | # @param message [String] Warning message to send 17 | # @return [void] 18 | def self.log_warn(message) 19 | notify('⚠️ WARNING', message, 16_771_899, :logs_webhook) 20 | end 21 | 22 | # Sends a warning log message to Discord 23 | # @param message [String] Warning message to send 24 | # @return [void] 25 | def self.log_error(message) 26 | notify('⚠️ ERROR', message, 14_549_051, :logs_webhook) 27 | end 28 | 29 | # Sends an informational log message to Discord 30 | # @param message [String] Info message to send 31 | # @return [void] 32 | def self.log_info(message) 33 | notify('ℹ️ INFO', message, 5_025_616, :logs_webhook) 34 | end 35 | 36 | # Notifies about a new program addition 37 | # @param platform [String] Platform name 38 | # @param title [String] Program title 39 | # @param _slug [String] Program slug (unused) 40 | # @param _private [Boolean] Whether the program is private (unused) 41 | # @return [void] 42 | def self.new_program(platform, title, _slug, _private) 43 | notify('🆕 Program add', "The program '#{title}' has been added to #{platform}", 5_025_616) 44 | end 45 | 46 | # Notifies about a program removal 47 | # @param platform [String] Platform name 48 | # @param title [String] Program title 49 | # @return [void] 50 | def self.removed_program(platform, title) 51 | notify('🗑 Program removed', "The program '#{title}' has been removed from #{platform}", 16_711_680) 52 | end 53 | 54 | # Notifies about a new scope addition 55 | # @param platform [String] Platform name 56 | # @param program [String] Program title 57 | # @param value [String] Scope value 58 | # @param category [String] Scope category 59 | # @param in_scope [Boolean] Whether the scope is in-scope (true) or out-of-scope (false) 60 | # @return [void] 61 | def self.new_scope(platform, program, value, category, in_scope) 62 | # Do not send notifications for disabled categories 63 | return unless config[:notify_categories] == 'all' || config[:notify_categories].split(',').include?(category) 64 | 65 | notify('🆕 New scope', "In #{platform} - Program '#{program}': #{scope_label(value, category, in_scope)} added", 66 | 65_280) 67 | end 68 | 69 | # Notifies about a scope removal 70 | # @param platform [String] Platform name 71 | # @param program [String] Program title 72 | # @param value [String] Scope value 73 | # @param category [String] Scope category 74 | # @param in_scope [Boolean] Whether the scope is in-scope (true) or out-of-scope (false) 75 | # @return [void] 76 | def self.removed_scope(platform, program, value, category, in_scope) 77 | # Do not send notifications for disabled categories 78 | return unless config[:notify_categories] == 'all' || config[:notify_categories].split(',').include?(category) 79 | 80 | notify('🗑 Scope removed', 81 | "In #{platform} - Program '#{program}': #{scope_label(value, category, in_scope)} removed", 16_711_680) 82 | end 83 | 84 | # Creates a formatted scope label 85 | # @param value [String] Scope value 86 | # @param category [String] Scope category 87 | # @param in_scope [Boolean] Whether the scope is in-scope 88 | # @return [String] Formatted scope label 89 | def self.scope_label(value, category, in_scope) 90 | scope_type = in_scope ? 'In Scope' : 'Out of Scope' 91 | "[#{scope_type}] (#{category}) #{value}" 92 | end 93 | 94 | # Sends a notification to Discord through a webhook 95 | # @param title [String] Notification title 96 | # @param description [String] Notification description 97 | # @param color [Integer] Embed color 98 | # @param webhook [Symbol] Webhook to use (:message_webhook or :logs_webhook) 99 | # @return [void] 100 | def self.notify(title, description, color, webhook = :message_webhook) 101 | return unless config[webhook] && !config[webhook].empty? 102 | 103 | embed = { title: title, description: description, color: color } 104 | body = { embeds: [embed] }.to_json 105 | 106 | resp = HttpClient.post(config[webhook], { headers: config[:headers], body: body }) 107 | 108 | ratelimit_remaining = resp.headers['x-ratelimit-remaining']&.to_i 109 | ratelimit_reset_after = resp.headers['x-ratelimit-reset-after']&.to_i 110 | 111 | sleep(ratelimit_reset_after) if ratelimit_remaining&.zero? 112 | end 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /libs/utilities/parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'ipaddr' 4 | require 'uri' 5 | require 'yaml' 6 | 7 | module ScopesExtractor 8 | # Parser module provides utilities for parsing and validating various data formats 9 | # including JSON, IP addresses, and URIs 10 | module Parser 11 | class << self 12 | # Loads the exclusions from the YAML file 13 | # Uses memoization to avoid loading the file multiple times 14 | # @return [Array<String>] List of exclusion patterns 15 | def exclusions 16 | @exclusions ||= begin 17 | config_path = File.join(File.dirname(__FILE__), '..', '..', 'config', 'exclusions.yml') 18 | if File.exist?(config_path) 19 | config = YAML.safe_load(File.read(config_path)) 20 | config['exclusions'] || [] 21 | else 22 | [] 23 | end 24 | end 25 | end 26 | 27 | # Parses a JSON string into a Ruby object 28 | # @param data [String] JSON string to parse 29 | # @return [Hash, Array, nil] Parsed JSON object or nil if parsing fails 30 | def json_parse(data) 31 | JSON.parse(data) 32 | rescue JSON::ParserError 33 | Discord.log_warn("JSON parsing error : #{data}") 34 | nil 35 | end 36 | 37 | # Validates if a string represents a valid IP address 38 | # @param value [String] The IP address to validate 39 | # @return [Boolean] True if the value is a valid IP address, false otherwise 40 | def valid_ip?(value) 41 | IPAddr.new(value) 42 | true 43 | rescue IPAddr::InvalidAddressError 44 | Discord.log_warn("Bad IPAddr for '#{value}'") 45 | false 46 | end 47 | 48 | # Validates if a string represents a valid URI 49 | # @param value [String] The URI to validate 50 | # @return [Boolean] True if the value is a valid URI, false otherwise 51 | def valid_uri?(value) 52 | return false if exclusions.any? { |exclusion| value.include?(exclusion) } 53 | 54 | url = value.start_with?('http') ? value : "http://#{value.sub('*.', '')}" 55 | 56 | !!URI.parse(url)&.host 57 | rescue URI::InvalidURIError 58 | Discord.log_warn("Bad URI for '#{value}'") 59 | false 60 | end 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /libs/utilities/scopes_comparator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ScopesExtractor 4 | module Utilities 5 | # The ScopeComparator module provides functionality for comparing bug bounty program scopes 6 | # and detecting changes (additions and removals) between program versions 7 | module ScopeComparator 8 | # Compares old and new data, and triggers notifications for changes 9 | # @param old_data [Hash] Previous program scope data 10 | # @param new_data [Hash] Current program scope data 11 | # @param platforms [Array<String>] List of platforms to compare 12 | # @return [void] 13 | def self.compare_and_notify(old_data, new_data, platforms) 14 | parsed_new_data = Parser.json_parse(JSON.generate(new_data)) 15 | platforms.each do |platform| 16 | old_programs = old_data[platform] || {} 17 | next if old_programs.empty? 18 | 19 | new_programs = parsed_new_data[platform] || {} 20 | 21 | process_existing_and_new_programs(old_programs, new_programs, platform) 22 | process_removed_programs(old_programs, new_programs, platform) 23 | end 24 | end 25 | 26 | # Processes existing and new programs to detect changes and additions 27 | # @param old_programs [Hash] Previous program data 28 | # @param new_programs [Hash] Current program data 29 | # @param platform [String] The platform name 30 | # @return [void] 31 | def self.process_existing_and_new_programs(old_programs, new_programs, platform) 32 | new_programs.each do |title, info| 33 | if old_programs.key?(title) 34 | # For existing programs, compare scopes 35 | old_scopes = old_programs[title]['scopes'] || {} 36 | new_scopes = info['scopes'] || {} 37 | compare_scopes(new_scopes, old_scopes, title, platform) 38 | else 39 | # New program discovered 40 | Discord.new_program(platform, title, info['slug'], info['private']) 41 | # Also record in history 42 | DB.save_change(platform, title, 'add_program', nil, nil, title) 43 | end 44 | end 45 | end 46 | 47 | # Processes removed programs to detect deletions 48 | # @param old_programs [Hash] Previous program data 49 | # @param new_programs [Hash] Current program data 50 | # @param platform [String] The platform name 51 | # @return [void] 52 | def self.process_removed_programs(old_programs, new_programs, platform) 53 | old_programs.each_key do |title| 54 | next if new_programs.key?(title) 55 | 56 | Discord.removed_program(platform, title) 57 | # Also record in history 58 | DB.save_change(platform, title, 'remove_program', nil, nil, title) 59 | end 60 | end 61 | 62 | # Compares program scopes (in and out of scope) and notifies additions and deletions 63 | # @param new_scopes [Hash] Current program scope data 64 | # @param old_scopes [Hash] Previous program scope data 65 | # @param program_title [String] Program title 66 | # @param platform [String] The platform name 67 | # @return [void] 68 | def self.compare_scopes(new_scopes, old_scopes, program_title, platform) 69 | %w[in out].each do |scope_type| 70 | new_scope_groups = new_scopes[scope_type] || {} 71 | old_scope_groups = old_scopes[scope_type] || {} 72 | 73 | context = ScopeContext.new(program_title, platform, scope_type) 74 | detect_added_scopes(new_scope_groups, old_scope_groups, context) 75 | detect_removed_scopes(new_scope_groups, old_scope_groups, context) 76 | end 77 | end 78 | 79 | # Detects scopes that have been added 80 | # @param new_scope_groups [Hash] Current scope groups 81 | # @param old_scope_groups [Hash] Previous scope groups 82 | # @param context [ScopeContext] Context object containing program information 83 | # @return [void] 84 | def self.detect_added_scopes(new_scope_groups, old_scope_groups, context) 85 | new_scope_groups.each do |category, scopes_array| 86 | scopes_array.each do |scope| 87 | next if old_scope_groups[category]&.include?(scope) 88 | 89 | Discord.new_scope(context.platform, context.program_title, scope, category, context.in_scope?) 90 | # Also record in history 91 | DB.save_change( 92 | context.platform, 93 | context.program_title, 94 | 'add_scope', 95 | context.scope_type, 96 | category, 97 | scope 98 | ) 99 | end 100 | end 101 | end 102 | 103 | # Detects scopes that have been removed 104 | # @param new_scope_groups [Hash] Current scope groups 105 | # @param old_scope_groups [Hash] Previous scope groups 106 | # @param context [ScopeContext] Context object containing program information 107 | # @return [void] 108 | def self.detect_removed_scopes(new_scope_groups, old_scope_groups, context) 109 | old_scope_groups.each do |category, scopes_array| 110 | scopes_array.each do |scope| 111 | next if new_scope_groups[category]&.include?(scope) 112 | 113 | Discord.removed_scope(context.platform, context.program_title, scope, category, context.in_scope?) 114 | # Also record in history 115 | DB.save_change( 116 | context.platform, 117 | context.program_title, 118 | 'remove_scope', 119 | context.scope_type, 120 | category, 121 | scope 122 | ) 123 | end 124 | end 125 | end 126 | 127 | # ScopeContext encapsulates context information needed for scope comparison operations 128 | # @attr_reader program_title [String] The title of the program 129 | # @attr_reader platform [String] The platform name 130 | # @attr_reader scope_type [String] The scope type ('in' or 'out') 131 | class ScopeContext 132 | attr_reader :program_title, :platform, :scope_type 133 | 134 | # Initializes a new ScopeContext instance 135 | # @param program_title [String] The title of the program 136 | # @param platform [String] The platform name 137 | # @param scope_type [String] The scope type ('in' or 'out') 138 | def initialize(program_title, platform, scope_type) 139 | @program_title = program_title 140 | @platform = platform 141 | @scope_type = scope_type 142 | end 143 | 144 | # Determines if the context represents an in-scope item 145 | # @return [Boolean] true if in scope, false otherwise 146 | def in_scope? 147 | scope_type == 'in' 148 | end 149 | end 150 | end 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'simplecov' 4 | require 'simplecov-lcov' 5 | 6 | SimpleCov::Formatter::LcovFormatter.config.output_directory = 'coverage' 7 | SimpleCov::Formatter::LcovFormatter.config.report_with_single_file = true 8 | 9 | SimpleCov.formatter = SimpleCov::Formatter::LcovFormatter 10 | SimpleCov.start 11 | 12 | require 'webmock/rspec' 13 | require_relative '../libs/scopes_extractor' 14 | 15 | WebMock.disable_net_connect!(allow_localhost: true) 16 | 17 | RSpec.configure do |config| 18 | config.before(:each) do 19 | stub_request(:post, %r{discord\.com/api/webhooks}) 20 | end 21 | 22 | config.expect_with :rspec do |expectations| 23 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 24 | end 25 | 26 | config.mock_with :rspec do |mocks| 27 | mocks.verify_partial_doubles = true 28 | end 29 | 30 | config.shared_context_metadata_behavior = :apply_to_host_groups 31 | config.filter_run_when_matching :focus 32 | config.warnings = true 33 | config.order = :random 34 | Kernel.srand config.seed 35 | end 36 | -------------------------------------------------------------------------------- /spec/utilities/normalizer/bugcrowd_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe ScopesExtractor::Normalizer::Bugcrowd do 6 | describe '.normalization' do 7 | context 'when input contains domain with description after dash' do 8 | it 'extracts the domain part before the dash' do 9 | input = '*.domain.tld - lorem ipsum' 10 | expect(described_class.normalization(input)).to eq(['*.domain.tld']) 11 | end 12 | 13 | it 'handles multiple domain formats with descriptions' do 14 | inputs_and_expected = { 15 | 'domain.tld - lorem ipsum' => ['domain.tld'], 16 | 'https://api.domain.tld - API endpoints' => ['https://api.domain.tld'], 17 | 'sub.domain.tld - Multiple words after dash' => ['sub.domain.tld'] 18 | } 19 | 20 | inputs_and_expected.each do |input, expected| 21 | expect(described_class.normalization(input)).to eq(expected) 22 | end 23 | end 24 | end 25 | 26 | context 'when input does not contain a dash' do 27 | it 'returns the input unchanged' do 28 | clean_inputs = [ 29 | 'domain.tld', 30 | '*.domain.tld', 31 | 'https://api.domain.tld' 32 | ] 33 | 34 | clean_inputs.each do |input| 35 | expect(described_class.normalization(input)).to eq([input]) 36 | end 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /spec/utilities/normalizer/hackerone_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe ScopesExtractor::Normalizer::Hackerone do 6 | describe '.normalization' do 7 | context 'when input contains domains separated by comma' do 8 | it 'splits the input into separate domains' do 9 | input = 'example.tld,example.com' 10 | expected = ['example.tld', 'example.com'] 11 | expect(described_class.normalization(input)).to eq(expected) 12 | end 13 | end 14 | 15 | context 'when input does not contain a comma' do 16 | it 'returns the input as a single-element array' do 17 | clean_inputs = [ 18 | 'domain.tld', 19 | '*.domain.tld', 20 | 'https://api.domain.tld' 21 | ] 22 | 23 | clean_inputs.each do |input| 24 | expect(described_class.normalization(input.dup)).to eq([input]) 25 | end 26 | end 27 | end 28 | 29 | context 'domain.(TLD)' do 30 | it 'returns domain.com' do 31 | input = 'domain.(TLD)' 32 | expected = ['domain.com'] 33 | expect(described_class.normalization(input)).to eq(expected) 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/utilities/normalizer/intigriti_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe ScopesExtractor::Normalizer::Intigriti do 6 | describe '.normalization' do 7 | context 'when input contains domains separated by slash' do 8 | it 'splits the input into separate domains' do 9 | input = 'aaa.example.tld / login.example.tld / account.example.tld' 10 | expected = ['aaa.example.tld', 'login.example.tld', 'account.example.tld'] 11 | expect(described_class.normalization(input)).to eq(expected) 12 | end 13 | 14 | it 'handles various domain formats separated by slashes' do 15 | inputs_and_expected = { 16 | 'example.tld / example.xyz' => ['example.tld', 'example.xyz'], 17 | 'example.tld / example.xyz / example.com' => ['example.tld', 'example.xyz', 'example.com'] 18 | } 19 | 20 | inputs_and_expected.each do |input, expected| 21 | expect(described_class.normalization(input)).to eq(expected) 22 | end 23 | end 24 | end 25 | 26 | context 'when input contains wildcards' do 27 | it 'correctly formats wildcard domains' do 28 | inputs_and_expected = { 29 | '*. domain.tld' => ['*.domain.tld'], 30 | '* .domain.tld' => ['*.domain.tld'], 31 | '* domain.tld' => ['*.domain.tld'] 32 | } 33 | 34 | inputs_and_expected.each do |input, expected| 35 | expect(described_class.normalization(input.dup)).to eq(expected) 36 | end 37 | end 38 | end 39 | 40 | context 'when input does not contain a slash' do 41 | it 'returns the input as a single-element array' do 42 | clean_inputs = [ 43 | 'domain.tld', 44 | '*.domain.tld', 45 | 'https://api.domain.tld' 46 | ] 47 | 48 | clean_inputs.each do |input| 49 | expect(described_class.normalization(input.dup)).to eq([input]) 50 | end 51 | end 52 | end 53 | 54 | context 'domain.<TLD>' do 55 | it 'returns domain.com' do 56 | input = 'domain.<TLD>' 57 | expected = ['domain.com'] 58 | expect(described_class.normalization(input)).to eq(expected) 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /spec/utilities/normalizer/yeswehack_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe ScopesExtractor::Normalizer::YesWeHack do 6 | describe '.normalization' do 7 | context 'when input contains excluded patterns' do 8 | it 'returns an empty array' do 9 | excluded_inputs = [ 10 | 'endpoints on our sites', 11 | 'special scenarios', 12 | 'core services', 13 | 'see program description', 14 | 'see description' 15 | ] 16 | 17 | excluded_inputs.each do |input| 18 | expect(described_class.normalization(input)).to eq([]) 19 | end 20 | end 21 | end 22 | 23 | context 'when input contains multiple TLDs in parentheses' do 24 | it 'expands into multiple domains with different TLDs' do 25 | input = '*.example.(com|org|net)' 26 | expected = ['*.example.com', '*.example.org', '*.example.net'] 27 | expect(described_class.normalization(input)).to match_array(expected) 28 | end 29 | 30 | it 'handles https prefix correctly' do 31 | input = 'https://example.(com|org)' 32 | expected = ['https://example.com', 'https://example.org'] 33 | expect(described_class.normalization(input)).to match_array(expected) 34 | end 35 | end 36 | 37 | context 'when input does not match any special patterns' do 38 | it 'returns the input value unchanged' do 39 | input = 'example.com' 40 | expect(described_class.normalization(input)).to eq([input]) 41 | end 42 | end 43 | end 44 | 45 | describe '.excluded_pattern?' do 46 | it 'returns true when the value contains an excluded pattern' do 47 | expect(described_class.excluded_pattern?('This is about core services and more')).to be true 48 | end 49 | 50 | it 'returns false when the value does not contain an excluded pattern' do 51 | expect(described_class.excluded_pattern?('example.com')).to be false 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /spec/utilities/parser_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | RSpec.describe ScopesExtractor::Parser do 6 | describe '.exclusions' do 7 | context 'when exclusions.yml does not exist' do 8 | before do 9 | allow(File).to receive(:exist?).and_return(false) 10 | 11 | # Reset memoized value 12 | described_class.instance_variable_set(:@exclusions, nil) 13 | end 14 | 15 | it 'returns an empty array' do 16 | expect(described_class.exclusions).to eq([]) 17 | end 18 | end 19 | 20 | context 'when exclusions.yml exists but has no exclusions key' do 21 | let(:yaml_content) { { 'other_key' => ['value'] } } 22 | let(:yaml_path) { File.join(File.dirname(__FILE__), '..', '..', 'config', 'exclusions.yml') } 23 | 24 | before do 25 | allow(File).to receive(:exist?).with(anything).and_call_original 26 | allow(File).to receive(:exist?).with(yaml_path).and_return(true) 27 | allow(File).to receive(:read).with(yaml_path).and_return(yaml_content.to_yaml) 28 | 29 | # Reset memoized value 30 | described_class.instance_variable_set(:@exclusions, nil) 31 | end 32 | end 33 | end 34 | 35 | describe '.json_parse' do 36 | context 'with valid JSON' do 37 | it 'parses JSON objects correctly' do 38 | json_string = '{"key": "value", "number": 42}' 39 | expected = { 'key' => 'value', 'number' => 42 } 40 | expect(described_class.json_parse(json_string)).to eq(expected) 41 | end 42 | 43 | it 'parses JSON arrays correctly' do 44 | json_string = '[1, 2, 3, "test"]' 45 | expected = [1, 2, 3, 'test'] 46 | expect(described_class.json_parse(json_string)).to eq(expected) 47 | end 48 | end 49 | 50 | context 'with invalid JSON' do 51 | it 'returns nil and logs a warning' do 52 | invalid_json = '{"broken": "json' 53 | expect(described_class.json_parse(invalid_json)).to be_nil 54 | end 55 | end 56 | end 57 | 58 | describe '.valid_ip?' do 59 | context 'with valid IP addresses' do 60 | it 'returns true for valid IPv4 addresses' do 61 | expect(described_class.valid_ip?('192.168.1.1')).to be true 62 | expect(described_class.valid_ip?('10.0.0.1')).to be true 63 | expect(described_class.valid_ip?('127.0.0.1')).to be true 64 | end 65 | 66 | it 'returns true for valid IPv6 addresses' do 67 | expect(described_class.valid_ip?('::1')).to be true 68 | expect(described_class.valid_ip?('2001:db8::1')).to be true 69 | expect(described_class.valid_ip?('fe80::1')).to be true 70 | end 71 | end 72 | 73 | context 'with invalid IP addresses' do 74 | it 'returns false and logs a warning' do 75 | invalid_ip = '256.256.256.256' 76 | expect(described_class.valid_ip?(invalid_ip)).to be false 77 | end 78 | 79 | it 'returns false for non-IP strings' do 80 | invalid_ip = 'not-an-ip' 81 | expect(described_class.valid_ip?(invalid_ip)).to be false 82 | end 83 | end 84 | end 85 | 86 | describe '.valid_uri?' do 87 | before do 88 | # Mock exclusions method to return test values 89 | allow(described_class).to receive(:exclusions).and_return(['excluded.com']) 90 | end 91 | 92 | context 'with valid URIs' do 93 | it 'returns true for valid URIs with scheme' do 94 | expect(described_class.valid_uri?('https://example.com')).to be true 95 | expect(described_class.valid_uri?('http://sub.domain.org/path')).to be true 96 | end 97 | 98 | it 'returns true for valid hostnames without scheme' do 99 | expect(described_class.valid_uri?('example.com')).to be true 100 | expect(described_class.valid_uri?('sub.example.org')).to be true 101 | end 102 | end 103 | 104 | context 'with a wildcard' do 105 | it 'returns true for valid wildcard without scheme' do 106 | expect(described_class.valid_uri?('*.example.com')).to be true 107 | end 108 | end 109 | 110 | context 'with invalid URIs' do 111 | it 'returns false for excluded URIs' do 112 | expect(described_class.valid_uri?('excluded.com')).to be false 113 | end 114 | 115 | it 'returns false and logs a warning for invalid URIs' do 116 | invalid_uri = 'http://exa mple.com' 117 | expect(described_class.valid_uri?(invalid_uri)).to be false 118 | end 119 | end 120 | end 121 | end 122 | --------------------------------------------------------------------------------