├── .idea ├── .gitignore ├── codeStyles │ └── codeStyleConfig.xml ├── modules.xml ├── quickcert.iml └── vcs.xml ├── README.md ├── go.mod ├── go.sum ├── quickcert └── quickcert.go /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/quickcert.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # QuickCert 2 | 3 | QuickCert is a high-performance tool for extracting subdomains from SSL/TLS certificate transparency logs using crt.sh's PostgreSQL database. Unlike traditional HTTP API methods, QuickCert offers improved reliability and unlimited result retrieval by directly connecting to the certificate transparency database. 4 | 5 | ## Features 6 | 7 | - 🚀 Direct PostgreSQL connection to crt.sh database 8 | - 💪 Multi-threaded processing (10 concurrent connections) 9 | - 🔄 Automatic retry mechanism for failed queries 10 | - 🎯 Smart duplicate filtering 11 | - ⚡ High-performance using pgx driver 12 | - 📝 Case-insensitive matching 13 | - 🧹 Automatic wildcard certificate handling 14 | 15 | ## Installation 16 | 17 | ### Using Go Install 18 | ```bash 19 | go install github.com/c3l3si4n/quickcert@HEAD 20 | ``` 21 | 22 | ### Building from Source 23 | ```bash 24 | git clone https://github.com/c3l3si4n/quickcert.git 25 | cd quickcert 26 | go build 27 | ``` 28 | 29 | ## Usage 30 | 31 | ### Basic Usage 32 | ```bash 33 | echo "example.com" | quickcert 34 | ``` 35 | 36 | ### Multiple Domains 37 | ```bash 38 | cat domains.txt | quickcert 39 | ``` 40 | 41 | ### Combining with Other Tools 42 | ```bash 43 | echo "example.com" | quickcert | tee subdomains.txt 44 | ``` 45 | 46 | ## Technical Details 47 | 48 | - Database: Connects to crt.sh PostgreSQL database (certwatch) 49 | - Connection String: `postgres://guest@crt.sh:5432/certwatch` 50 | - Query Limit: 15,000 records per page 51 | - Retry Mechanism: Up to 5 retries per failed query 52 | - Concurrent Connections: 10 parallel queries 53 | 54 | ## Features in Detail 55 | 56 | 1. **Duplicate Handling** 57 | - Automatically removes duplicate subdomains 58 | - Converts all domains to lowercase for consistent matching 59 | 60 | 2. **Wildcard Certificate Processing** 61 | - Automatically strips `*.` from wildcard certificates 62 | - Ensures proper subdomain formatting 63 | 64 | 3. **Error Handling** 65 | - Graceful handling of database connection issues 66 | - Automatic query retries on failure 67 | - Concurrent connection management 68 | 69 | ## Limitations 70 | 71 | - Fixed number of concurrent connections (10) 72 | - Dependent on crt.sh database availability 73 | 74 | ## Contributing 75 | 76 | Contributions are welcome! Please feel free to submit a Pull Request. 77 | 78 | ## Acknowledgments 79 | 80 | - crt.sh for providing public access to their certificate transparency database 81 | - The Go community for excellent database drivers and tools 82 | 83 | ## Author 84 | 85 | [c3l3si4n](https://github.com/c3l3si4n) 86 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/c3l3si4n/quickcert 2 | 3 | go 1.21 4 | 5 | require github.com/jackc/pgx/v5 v5.4.3 6 | 7 | require ( 8 | github.com/jackc/pgpassfile v1.0.0 // indirect 9 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 10 | golang.org/x/crypto v0.9.0 // indirect 11 | golang.org/x/text v0.9.0 // indirect 12 | ) 13 | 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 5 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 6 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= 7 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 8 | github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= 9 | github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= 10 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 11 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 13 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 14 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 15 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 16 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 17 | golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= 18 | golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= 19 | golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= 20 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 21 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 22 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 23 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 24 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 25 | -------------------------------------------------------------------------------- /quickcert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/c3l3si4n/quickcert/ffe5f3e6a88a9a2bacd3fcdd82dcda2d92adb85b/quickcert -------------------------------------------------------------------------------- /quickcert.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | "sync" 11 | 12 | "github.com/jackc/pgx/v5" 13 | ) 14 | 15 | // psql -h crt.sh -p 5432 -U guest certwatch 16 | var CRTSH_DATABASE_URL = "postgres://guest@crt.sh:5432/certwatch?sslmode=disable&default_query_exec_mode=simple_protocol" 17 | 18 | func IterStdin() []string { 19 | out := []string{} 20 | scanner := bufio.NewScanner(os.Stdin) 21 | for scanner.Scan() { 22 | out = append(out, scanner.Text()) 23 | } 24 | 25 | return out 26 | } 27 | 28 | var Limit = 15000 29 | 30 | func main() { 31 | uniqueMap := make(map[string]bool) 32 | uniqueMapMutex := sync.Mutex{} 33 | 34 | query := ` 35 | WITH ci AS ( 36 | SELECT 37 | x509_commonName(sub.CERTIFICATE) COMMON_NAME 38 | 39 | 40 | FROM (SELECT cai.CERTIFICATE CERTIFICATE 41 | FROM certificate_and_identities cai 42 | WHERE plainto_tsquery('certwatch', '%s') @@ identities(cai.CERTIFICATE) 43 | AND cai.NAME_VALUE ILIKE ('%%' || '%s') 44 | LIMIT %d OFFSET %d 45 | ) sub 46 | 47 | GROUP BY sub.CERTIFICATE 48 | ) 49 | SELECT 50 | 51 | ci.COMMON_NAME COMMON_NAME 52 | 53 | FROM ci 54 | 55 | 56 | 57 | WHERE COMMON_NAME IS NOT NULL 58 | ` 59 | // iterate lines in stdin 60 | // for each line, prepare a query 61 | stdin := IterStdin() 62 | for _, line := range stdin { 63 | page := 0 64 | var routineQueue = make(chan bool, 10) 65 | stop := false 66 | var wait sync.WaitGroup 67 | lineCopy := line 68 | for { 69 | if stop { 70 | break 71 | } 72 | offset := Limit * page 73 | 74 | line := strings.ToLower(lineCopy) 75 | preparedQuery := fmt.Sprintf(query, line, line, Limit, offset) 76 | wait.Add(1) 77 | routineQueue <- true 78 | pageTmp := page 79 | go func(page int) { 80 | retries := 0 81 | success := false 82 | for !success && retries <= 5 { 83 | 84 | conn, err := pgx.Connect(context.Background(), CRTSH_DATABASE_URL) 85 | 86 | if err != nil { 87 | fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) 88 | os.Exit(1) 89 | } 90 | rows, err := conn.Query(context.Background(), preparedQuery) 91 | if err != nil { 92 | log.Println(err) 93 | 94 | rows, err = conn.Query(context.Background(), preparedQuery) 95 | if err != nil { 96 | retries += 1 97 | conn.Close(context.Background()) 98 | 99 | continue 100 | } 101 | } 102 | subdomains, err := pgx.CollectRows(rows, pgx.RowTo[string]) 103 | if err != nil { 104 | log.Println(err) 105 | 106 | subdomains, err = pgx.CollectRows(rows, pgx.RowTo[string]) 107 | if err != nil { 108 | retries += 1 109 | conn.Close(context.Background()) 110 | 111 | continue 112 | } 113 | } 114 | if len(subdomains) == 0 { 115 | stop = true 116 | <-routineQueue 117 | wait.Done() 118 | success = true 119 | conn.Close(context.Background()) 120 | 121 | return 122 | } 123 | for _, subdomain := range subdomains { 124 | subdomain = strings.ToLower(subdomain) 125 | if !strings.Contains(subdomain, line) { 126 | continue 127 | } 128 | if strings.HasPrefix(subdomain, "*.") { 129 | subdomain = strings.Replace(subdomain, "*.", "", 1) 130 | } 131 | uniqueMapMutex.Lock() 132 | if _, ok := uniqueMap[subdomain]; !ok { 133 | fmt.Println(subdomain) 134 | uniqueMap[subdomain] = true 135 | 136 | } 137 | uniqueMapMutex.Unlock() 138 | 139 | } 140 | conn.Close(context.Background()) 141 | success = true 142 | 143 | } 144 | wait.Done() 145 | <-routineQueue 146 | 147 | }(pageTmp) 148 | page += 1 149 | 150 | } 151 | 152 | wait.Wait() 153 | 154 | } 155 | } 156 | --------------------------------------------------------------------------------