├── grep-backURLs ├── go.mod ├── config.json ├── grep_keywords.txt └── main.go ├── LICENSE ├── grep_keywords.txt └── README.md /grep-backURLs/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | go 1.24.1 4 | -------------------------------------------------------------------------------- /grep-backURLs/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "domain": "", 3 | "output_dir": "output", 4 | "max_concurrency": 10, 5 | "timeout_seconds": 30000, 6 | "enable_logging": true, 7 | "enable_filtering": true, 8 | "custom_keywords": [], 9 | "timestamp": "2025-05-24T18:53:23.056393+05:30" 10 | } 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Dark Yagami 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /grep_keywords.txt: -------------------------------------------------------------------------------- 1 | # File Extensions 2 | \.zip$ 3 | \.exe$ 4 | \.pdf$ 5 | \.gz$ 6 | \.tar\.gz$ 7 | \.tar$ 8 | \.mp3$ 9 | \.wav$ 10 | \.doc$ 11 | \.jpg$ 12 | \.png$ 13 | \.txt$ 14 | \.aif$ 15 | \.aiff$ 16 | \.asd$ 17 | \.flac$ 18 | \.m4a$ 19 | \.m4p$ 20 | \.m4r$ 21 | \.wma$ 22 | \.DS_Store$ 23 | \.env$ 24 | \.config$ 25 | \.conf$ 26 | \.bak$ 27 | \.backup$ 28 | \.sql$ 29 | \.db$ 30 | \.sqlite$ 31 | \.pem$ 32 | \.key$ 33 | \.crt$ 34 | \.cer$ 35 | \.p12$ 36 | \.pfx$ 37 | \.log$ 38 | \.old$ 39 | \.swp$ 40 | \.yaml$ 41 | \.yml$ 42 | 43 | # Parameters and Endpoints 44 | [?&]id= 45 | [?&]username= 46 | [?&]password= 47 | [?&]uri= 48 | [?&]url= 49 | [?&]redirect= 50 | [?&]file= 51 | [?&]path= 52 | [?&]source= 53 | [?&]next= 54 | [?&]target= 55 | [?&]rurl= 56 | [?&]dest= 57 | [?&]destination= 58 | [?&]redir= 59 | [?&]redirect_uri= 60 | [?&]redirect_url= 61 | [?&]view= 62 | [?&]go= 63 | [?&]return= 64 | [?&]returnTo= 65 | [?&]return_to= 66 | [?&]checkout_url= 67 | [?&]continue= 68 | [?&]return_path= 69 | 70 | 71 | # Sensitive Paths 72 | /cgi-bin/redirect\.cgi 73 | /out/ 74 | /login 75 | /view 76 | /redirect/ 77 | 78 | # Git and SVN 79 | /\.git 80 | /\.git-rewrite 81 | /\.git/HEAD 82 | /\.git/index 83 | /\.git/logs 84 | /\.gitattributes 85 | /\.gitconfig 86 | /\.gitkeep 87 | /\.gitmodules 88 | /\.gitreview 89 | /\.svn/entries 90 | /\.svnignore 91 | 92 | # UTM Parameters 93 | utm_source 94 | utm_medium 95 | utm_campaign 96 | 97 | # Common Parameters 98 | page_id 99 | action 100 | share 101 | page 102 | view 103 | 104 | # Security Related 105 | accesskey 106 | admin 107 | aes 108 | api_key 109 | apikey 110 | checkClientTrusted 111 | crypt 112 | password 113 | pinning 114 | secret 115 | SHA256 116 | SharedPreferences 117 | superuser 118 | token 119 | X509TrustManager 120 | 121 | # Database Related 122 | myspl 123 | sql 124 | insert\s+into 125 | 126 | # File Extensions (Additional Sensitive Files) 127 | \.env$ 128 | \.config$ 129 | \.conf$ 130 | \.bak$ 131 | \.backup$ 132 | \.sql$ 133 | \.db$ 134 | \.sqlite$ 135 | \.pem$ 136 | \.key$ 137 | \.crt$ 138 | \.cer$ 139 | \.p12$ 140 | \.pfx$ 141 | \.log$ 142 | \.old$ 143 | \.swp$ 144 | \.yaml$ 145 | \.yml$ 146 | 147 | # API and Authentication 148 | (?i)(api[_-]?key|access[_-]?token|secret[_-]?key)=[^&]* 149 | (?i)bearer\s+[a-zA-Z0-9\-\._~\+\/]+=* 150 | (?i)basic\s+[a-zA-Z0-9\-\._~\+\/]+=* 151 | (?i)auth[a-zA-Z0-9]*= 152 | (?i)jwt= 153 | 154 | # Sensitive Information 155 | (?i)[0-9]{16}(?:[0-9]{3})? # Credit Card Numbers 156 | (?i)[0-9]{3}-[0-9]{2}-[0-9]{4} # SSN 157 | (?i)[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,} # Email addresses 158 | 159 | # Development and Debug 160 | (?i)debug= 161 | (?i)test= 162 | (?i)dev[_-] 163 | (?i)staging[_-] 164 | (?i)phpinfo 165 | (?i)admin[_-] 166 | (?i)console 167 | (?i)swagger 168 | (?i)graphql 169 | 170 | # Database Queries 171 | (?i)(select|insert|update|delete|union|where)\s+.*\s+from 172 | (?i)database= 173 | (?i)dbname= 174 | (?i)mysql:// 175 | (?i)postgresql:// 176 | (?i)mongodb:// 177 | 178 | # Cloud Storage 179 | (?i)s3\.amazonaws\.com 180 | (?i)storage\.googleapis\.com 181 | (?i)blob\.core\.windows\.net 182 | (?i)firebasestorage\.googleapis\.com 183 | 184 | # Authentication Endpoints 185 | /oauth/ 186 | /auth/ 187 | /login/ 188 | /signin/ 189 | /signup/ 190 | /register/ 191 | /reset/ 192 | /forgot/ 193 | /password/ 194 | /session/ 195 | 196 | # User Related 197 | (?i)user[_-]?id= 198 | (?i)account[_-]?id= 199 | (?i)profile[_-]?id= 200 | (?i)member[_-]?id= 201 | (?i)customer[_-]?id= 202 | (?i)admin[_-]?id= 203 | 204 | # Server Information 205 | (?i)server[_-] 206 | (?i)host[_-] 207 | (?i)domain[_-] 208 | (?i)port= 209 | (?i)protocol= 210 | 211 | # Common Vulnerabilities 212 | (?i)(\.\.\/|\.\.\\) # Directory Traversal 213 | (?i) 627 | 628 | 629 | ` 630 | // ReportData struct for HTML template 631 | type ReportData struct { 632 | Domain string 633 | GeneratedTime string 634 | Results []PatternResult 635 | } 636 | 637 | reportData := ReportData{ 638 | Domain: domain, 639 | GeneratedTime: currentTimestamp(), 640 | Results: resultsData, 641 | } 642 | 643 | tmpl, err := template.New("report").Funcs(template.FuncMap{ 644 | "sub": func(a, b int) int { return a - b }, 645 | "base": filepath.Base, 646 | }).Parse(htmlTemplate) 647 | if err != nil { 648 | return fmt.Errorf("error parsing HTML template: %v", err) 649 | } 650 | 651 | file, err := os.Create(reportFilePath) 652 | if err != nil { 653 | return fmt.Errorf("error creating HTML report file: %v", err) 654 | } 655 | defer file.Close() 656 | 657 | if err := tmpl.Execute(file, reportData); err != nil { 658 | return fmt.Errorf("error executing HTML template: %v", err) 659 | } 660 | 661 | logMessage("INFO", fmt.Sprintf("HTML report generated at: %s", reportFilePath)) 662 | return nil 663 | } 664 | 665 | func printBanner() { 666 | banner := fmt.Sprintf(`%s 667 | ╔══════════════════════════════════════════════════════════════╗ 668 | ║ %sEnhanced URL Reconnaissance Tool%s ║ 669 | ║ %s v%s %s ║ 670 | ╚══════════════════════════════════════════════════════════════╝ 671 | %s`, colorCyan, colorBold, colorReset+colorCyan, colorBold, version, colorReset+colorCyan, colorReset) 672 | fmt.Println(banner) 673 | } 674 | 675 | // printUsage displays the tool's usage instructions and options. 676 | func printUsage() { 677 | fmt.Printf(`%sUsage:%s 678 | %s [options]%s 679 | 680 | %sA tool to find sensitive information by enumerating subdomains, collecting Wayback Machine URLs, 681 | analyzing them, and matching against custom patterns.%s 682 | 683 | %sOptions:%s 684 | `, colorYellow, colorReset, os.Args[0], colorReset, colorReset, colorReset, colorGreen, colorReset) 685 | flag.PrintDefaults() 686 | fmt.Printf(` 687 | %sExamples:%s 688 | %s -domain example.com -html 689 | %s -domain target.com -keywords-file custom_keywords.txt -json -markdown 690 | %s -domain example.org 691 | %s -v 692 | %s --config (to generate a default config.json or modify existing) 693 | `, colorGreen, colorReset, os.Args[0], os.Args[0], os.Args[0], os.Args[0], os.Args[0]) 694 | } 695 | 696 | // main function is the entry point of the program. 697 | func main() { 698 | var domainFlag string 699 | var keywordsFileFlag string 700 | var outputDirFlag string 701 | var generateJSONFlag bool 702 | var generateMarkdownFlag bool 703 | var generateHTMLFlag bool 704 | var showVersionFlag bool 705 | var configSetupFlag bool 706 | 707 | // Define flags 708 | flag.StringVar(&domainFlag, "domain", "", "Specify the target domain (e.g., example.com)") 709 | flag.StringVar(&keywordsFileFlag, "keywords-file", "grep_keywords.txt", "Path to a file containing grep-like keywords (one per line)") 710 | flag.StringVar(&outputDirFlag, "output-dir", "output", "Base directory to store all scan output files") 711 | flag.BoolVar(&generateJSONFlag, "json", false, "Generate results in JSON format for each pattern") 712 | flag.BoolVar(&generateMarkdownFlag, "markdown", false, "Generate results in Markdown format for each pattern") 713 | flag.BoolVar(&generateHTMLFlag, "html", false, "Generate a comprehensive HTML report summarizing all findings in the current directory") 714 | flag.BoolVar(&showVersionFlag, "version", false, "Display the tool version and exit") 715 | flag.BoolVar(&showVersionFlag, "v", false, "Display the tool version and exit (shorthand)") 716 | flag.BoolVar(&configSetupFlag, "config", false, "Run interactive configuration setup and exit") 717 | 718 | flag.Usage = printUsage // Set custom usage function 719 | 720 | flag.Parse() // Parse command-line arguments here 721 | 722 | if showVersionFlag { 723 | fmt.Printf("grep-backURLs Version: %s\n", version) 724 | return 725 | } 726 | 727 | if configSetupFlag { 728 | logMessage("INFO", "Starting interactive configuration setup...") 729 | if err := loadConfig(); err != nil { // loadConfig will now also save the updated timestamp 730 | logMessage("ERROR", fmt.Sprintf("Failed to load/create config: %v", err)) 731 | return 732 | } 733 | logMessage("INFO", "Configuration setup complete. Modify config.json manually if needed.") 734 | return 735 | } 736 | 737 | // Load configuration (will also update config.Timestamp and save it) 738 | if err := loadConfig(); err != nil { 739 | logMessage("ERROR", fmt.Sprintf("Error loading config: %v", err)) 740 | return 741 | } 742 | 743 | // Store the original config.OutputDir loaded from the file 744 | originalConfigOutputDir := config.OutputDir 745 | 746 | // --- CRITICAL MODIFICATION: Check and STOP if -output-dir flag overrides config --- 747 | var outputDirFlagWasProvided bool 748 | flag.Visit(func(f *flag.Flag) { 749 | if f.Name == "output-dir" { 750 | outputDirFlagWasProvided = true 751 | } 752 | }) 753 | 754 | if outputDirFlagWasProvided { 755 | if outputDirFlag != originalConfigOutputDir { 756 | logMessage("ERROR", fmt.Sprintf("Command-line flag '-output-dir %s' differs from 'output_dir' in config.json ('%s').", outputDirFlag, originalConfigOutputDir)) 757 | logMessage("ERROR", "Please update 'output_dir' in your config.json to match the desired output directory, then run the tool again.") 758 | logMessage("ERROR", "Exiting to ensure consistent configuration.") 759 | return // Exit the program 760 | } 761 | } 762 | // If the flag was provided and matches, or if it wasn't provided, 763 | // config.OutputDir will already hold the correct value (either from flag or config). 764 | // No explicit assignment needed here unless you want to force the flag value even if it matches config. 765 | if outputDirFlagWasProvided { 766 | config.OutputDir = outputDirFlag 767 | } 768 | // --- END CRITICAL MODIFICATION --- 769 | 770 | // Override domain setting with command-line flag if provided 771 | if domainFlag != "" { 772 | config.Domain = domainFlag 773 | } 774 | 775 | // Prompt for domain if not provided via flag or config 776 | if config.Domain == "" { 777 | fmt.Printf("%sEnter the target domain (e.g., example.com): %s", colorBold, colorReset) 778 | fmt.Scanln(&config.Domain) 779 | } 780 | 781 | // Final check for domain 782 | if config.Domain == "" { 783 | logMessage("ERROR", "Domain cannot be empty. Exiting.") 784 | flag.Usage() 785 | return 786 | } 787 | results.Domain = config.Domain // Set domain in results struct 788 | 789 | // Create output directory 790 | if err := os.MkdirAll(config.OutputDir, 0755); err != nil { 791 | logMessage("ERROR", fmt.Sprintf("Error creating output directory %s: %v", config.OutputDir, err)) 792 | return 793 | } 794 | logMessage("INFO", fmt.Sprintf("All output will be saved to: %s", config.OutputDir)) 795 | 796 | // Setup logging to file 797 | if err := setupLogging(config.Domain); err != nil { 798 | logMessage("WARN", fmt.Sprintf("Failed to set up logging: %v. Proceeding without file logging.", err)) 799 | } 800 | defer func() { 801 | if logFile != nil { 802 | logFile.Close() // Ensure log file is closed on exit 803 | } 804 | }() 805 | 806 | // --- MODIFICATION: Define file paths for subfinder and waybackurls output in the current working directory --- 807 | cwd, _ := os.Getwd() 808 | subFile := filepath.Join(cwd, fmt.Sprintf("%s_subdomains.txt", config.Domain)) 809 | waybackFile := filepath.Join(cwd, fmt.Sprintf("%s_waybackurls.txt", config.Domain)) 810 | // --- END MODIFICATION --- 811 | 812 | // Check if keywords file exists 813 | if _, err := os.Stat(keywordsFileFlag); os.IsNotExist(err) { 814 | logMessage("ERROR", fmt.Sprintf("Keywords file '%s' not found. Please create it with one keyword per line.", keywordsFileFlag)) 815 | return 816 | } 817 | 818 | var wg sync.WaitGroup 819 | 820 | // --- MODIFICATION: Use Subfinder only --- 821 | wg.Add(1) 822 | go func() { 823 | defer wg.Done() 824 | cmd := exec.Command("subfinder", "-d", config.Domain, "-o", subFile) 825 | if err := runCommand(cmd, "Subfinder"); err != nil { 826 | logMessage("ERROR", fmt.Sprintf("Subfinder failed for %s: %v. Cannot proceed without subdomains. Exiting.", config.Domain, err)) 827 | os.Exit(1) // Exit if subfinder fails, as waybackurls depends on it 828 | } 829 | }() 830 | wg.Wait() // Wait for Subfinder to complete 831 | 832 | // Check if subdomains were found 833 | if isEmptyFile(subFile) { 834 | logMessage("ERROR", fmt.Sprintf("No subdomains found in '%s' or Subfinder failed to populate it. Cannot proceed with Waybackurls. Exiting.", subFile)) 835 | return 836 | } 837 | 838 | // Read subdomains and update results 839 | subContent, err := os.ReadFile(subFile) 840 | if err != nil { 841 | logMessage("ERROR", fmt.Sprintf("Error reading subdomains file '%s': %v", subFile, err)) 842 | return 843 | } 844 | results.SubdomainCount = len(strings.Split(strings.TrimSpace(string(subContent)), "\n")) 845 | if results.SubdomainCount == 1 && strings.TrimSpace(string(subContent)) == "" { // Handle case of empty file but split gives 1 empty string 846 | results.SubdomainCount = 0 847 | } 848 | logMessage("INFO", fmt.Sprintf("Found %d subdomains.", results.SubdomainCount)) 849 | 850 | // Run Waybackurls concurrently 851 | wg.Add(1) 852 | go func() { 853 | defer wg.Done() 854 | cmd := exec.Command("waybackurls") 855 | file, err := os.Open(subFile) 856 | if err != nil { 857 | logMessage("ERROR", fmt.Sprintf("Error opening subdomain file '%s' for waybackurls: %v", subFile, err)) 858 | return 859 | } 860 | defer file.Close() 861 | cmd.Stdin = file // Pipe subdomains to waybackurls stdin 862 | outFile, err := os.Create(waybackFile) 863 | if err != nil { 864 | logMessage("ERROR", fmt.Sprintf("Error creating wayback file '%s': %v", waybackFile, err)) 865 | return 866 | } 867 | defer outFile.Close() 868 | cmd.Stdout = outFile // Redirect waybackurls stdout to file 869 | if err := runCommand(cmd, "Waybackurls"); err != nil { 870 | logMessage("WARN", fmt.Sprintf("Waybackurls failed for %s: %v. Proceeding with potentially empty Wayback URLs file.", config.Domain, err)) 871 | } 872 | }() 873 | wg.Wait() // Wait for Waybackurls to complete 874 | 875 | // Read wayback URLs and update results 876 | waybackContent, err := os.ReadFile(waybackFile) 877 | if err != nil { 878 | logMessage("ERROR", fmt.Sprintf("Error reading wayback file '%s': %v", waybackFile, err)) 879 | return 880 | } 881 | results.URLCount = len(strings.Split(strings.TrimSpace(string(waybackContent)), "\n")) 882 | if results.URLCount == 1 && strings.TrimSpace(string(waybackContent)) == "" { // Handle case of empty file 883 | results.URLCount = 0 884 | } 885 | logMessage("INFO", fmt.Sprintf("Collected %d Wayback URLs.", results.URLCount)) 886 | 887 | logMessage("INFO", "Processing keywords and patterns...") 888 | keywordsContent, err := os.ReadFile(keywordsFileFlag) 889 | if err != nil { 890 | logMessage("ERROR", fmt.Sprintf("Error reading grep keywords file '%s': %v", keywordsFileFlag, err)) 891 | return 892 | } 893 | 894 | allKeywords := append(strings.Split(string(keywordsContent), "\n"), config.CustomKeywords...) 895 | 896 | allPatternResults, err := processKeywords(waybackContent, allKeywords, config.Domain) 897 | if err != nil { 898 | logMessage("ERROR", fmt.Sprintf("Error processing keywords: %v", err)) 899 | return 900 | } 901 | 902 | logMessage("INFO", "Generating reports...") 903 | if err := generateReport(config.Domain, allPatternResults, generateHTMLFlag); err != nil { 904 | logMessage("ERROR", fmt.Sprintf("Error generating reports: %v", err)) 905 | return 906 | } 907 | 908 | // Print final summary banner 909 | fmt.Printf(` 910 | %s╔══════════════════════════════════════════════════════════════╗ 911 | ║ %sSUMMARY%s ║ 912 | ╠════════════════════════════════════════════════════════════════╣ 913 | ║ Domain: %-42s ║ ║ 914 | ║ Subdomains Found: %-42d ║ ║ 915 | ║ URLs Collected: %-42d ║ ║ 916 | ║ Total Matches: %-42d ║ ║ 917 | ║ Output Directory: %-42s ║ ║ 918 | ╚══════════════════════════════════════════════════════════════╝%s 919 | 920 | %sAutomation script completed successfully!%s 921 | `, colorGreen, colorBold, colorReset+colorGreen, config.Domain, results.SubdomainCount, results.URLCount, 922 | results.Statistics["total_matches"], config.OutputDir, colorReset, colorBold, colorReset) 923 | } 924 | --------------------------------------------------------------------------------