├── LICENSE ├── README.md └── git-user.rb /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Patrick Hurd 2 | 3 | Commercial use of this software is prohibited. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction except to commercial use, this includes the rights to use, copy, modify, merge, publish, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | git-user.rb 2 | ====== 3 | Patrick Hurd, Coalfire Federal 4 | 5 | --- 6 | 7 | OSINT tool specifically for targetting developers. 8 | 9 | What you get: 10 | - Profile information 11 | - Commit authorship information 12 | - See options list for non-default output 13 | 14 | Setup 15 | ----- 16 | 17 | 1. `sudo apt install ruby` 18 | 2. `sudo gem install httparty` 19 | 3. `sudo apt install aha` (Required for mine output) 20 | 4. `sudo apt install whois` (Required for whois output) 21 | 4. [Add your GitHub username/password/token to your gitconfig](https://stackoverflow.com/a/51327559) if you plan on mining private repos 22 | 23 | Usage: 24 | ------ 25 | ``` 26 | Usage: git-user.rb [options] 27 | -h, --help Show this help banner 28 | 29 | -u, --user USERNAME User to gather info from 30 | -o, --organization ORGANIZATION Organization to scrape 31 | -r, --repo REPO The repo whom's contributors to scrape 32 | --local ABSOLUTE_PATH Perform scrape on a repo local to your filesystem 33 | --name NAME Name to refer to a --local repo in report filenames 34 | 35 | -a, --auth Authenticate with HTTP basic auth 36 | -t, --token TOKEN Use specified GitHub personal access token 37 | 38 | -s, --stackoverflow Try to find users' accounts on StackOverflow 39 | -p, --pwned Search for relevant data breaches using haveibeenpwned 40 | -e, --extra_checking Do extra checking on email addresses 41 | -m, --mine Mine the repo or user/organization's repos for secrets 42 | --whois Perform whois lookup on domains found in profile information 43 | -l, --loud Perform active recon on users (scrape their personal site) 44 | 45 | --html Output main report to an HTML document 46 | -w, --wordlist Generate wordlist for use in password attacks 47 | -c, --csv Export discovered accounts to a GoPhish-importable CSV file 48 | ``` 49 | 50 | Add the following line to your `.bashrc` or `.zshrc` if you're using zsh to enable argument autocompletion (optional): 51 | 52 | ```bash 53 | complete -W "--help --user --organization --repo --auth --token --stackoverflow --pwned --extra_checking --mine --html --wordlist --whois --loud --csv --local --name" git-user.rb 54 | ``` 55 | 56 | Example command: 57 | 58 | `./git-user.rb -t deadb33f... -o Coalfire-Research -r Git-Scrapers -s -p -e -m --html -c` 59 | 60 | If you have two-factor authentication enabled on your GitHub account, you will need to [create](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) and use an application token instead of your password (using `-t TOKEN` instead of `-a`). 61 | 62 | Repo mining will skip forked repos. 63 | 64 | How you can help: 65 | ----- 66 | Check out the issues 67 | -------------------------------------------------------------------------------- /git-user.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'rubygems' 3 | require 'httparty' 4 | require 'optparse' 5 | require 'cgi' 6 | require 'io/console' 7 | require 'json' 8 | require 'base64' 9 | 10 | API_ERR_MSG = "Invalid GitHub API response. Ensure you are authenticated, the rate limit has not been reached, and the resource you are looking for exists." 11 | pwHash = Hash.new(0) # global password list filled if :wl is enabled 12 | 13 | def check_auth_resp(resp) 14 | if !resp.is_a?(Array) 15 | STDERR.puts API_ERR_MSG 16 | STDERR.puts resp 17 | exit 18 | end 19 | end 20 | 21 | def printn(toprint) 22 | print "#{toprint}\n" 23 | end 24 | 25 | def printtn(toprint) 26 | print "\t#{toprint}\n" 27 | end 28 | 29 | def printttn(toprint) 30 | print "\t\t#{toprint}\n" 31 | end 32 | 33 | def pprinttn(key, value) 34 | print "\t#{key}\t#{value}\n" 35 | end 36 | 37 | def pprintttn(key, value) 38 | print "\t#{key}\t\t#{value}\n" 39 | end 40 | 41 | def printh(toprint) 42 | print "#{toprint}" 43 | bold = "\033[1;1m" 44 | reg = "\033[0;0m" 45 | filename = toprint[/(.\/report\/.*?html)/, 0] 46 | HTMLOut << "#{toprint}".gsub(bold, "").gsub(reg, "").gsub(/.\/report\/.*mined.html/, "#{filename}") 47 | end 48 | 49 | def printhn(toprint) 50 | print "#{toprint}\n" 51 | bold = "\033[1;1m" 52 | reg = "\033[0;0m" 53 | filename = toprint[/(.\/report\/.*?html)/, 0] 54 | HTMLOut << "#{toprint}\n".gsub(bold, "").gsub(reg, "").gsub(/.\/report\/.*mined.html/, "#{filename}") 55 | end 56 | 57 | def printhtn(toprint) 58 | print "\t#{toprint}\n" 59 | bold = "\033[1;1m" 60 | reg = "\033[0;0m" 61 | filename = toprint[/(.\/report\/.*?html)/, 0] 62 | HTMLOut << "\t#{toprint}\n".gsub(bold, "").gsub(reg, "").gsub(/.\/report\/.*mined.html/, "#{filename}") 63 | end 64 | 65 | def printhttn(toprint) 66 | print "\t\t#{toprint}\n" 67 | bold = "\033[1;1m" 68 | reg = "\033[0;0m" 69 | filename = toprint[/(.\/report\/.*?html)/, 0] 70 | HTMLOut << "\t\t#{toprint}\n".gsub(bold, "").gsub(reg, "").gsub(/.\/report\/.*mined.html/, "#{filename}") 71 | end 72 | 73 | def pprinthtn(key, value) 74 | print "\t#{key}\t#{value}\n" 75 | bold = "\033[1;1m" 76 | reg = "\033[0;0m" 77 | HTMLOut << "\t#{key}\t#{value}\n".gsub(bold, "").gsub(reg, "") 78 | end 79 | 80 | def pprinthttn(key, value) 81 | print "\t#{key}\t\t#{value}\n" 82 | bold = "\033[1;1m" 83 | reg = "\033[0;0m" 84 | HTMLOut << "\t#{key}\t\t#{value}\n".gsub(bold, "").gsub(reg, "") 85 | end 86 | 87 | def github_api_req(url, auth) 88 | resp = "" 89 | begin 90 | if auth[:token] 91 | resp = HTTParty.get(url, headers: { 92 | 'User-Agent' => UserAgent, 93 | 'Authorization' => "token #{auth[:token]}" 94 | }).body 95 | else 96 | resp = HTTParty.get(url, headers: { 97 | 'User-Agent' => UserAgent 98 | }, basic_auth: auth ).body 99 | end 100 | rescue => e 101 | STDERR.puts "Network error:\n#{e}\n" 102 | exit() 103 | end 104 | return resp 105 | end 106 | 107 | UserAgent = "Proprietary OSINT Tool" 108 | HTMLOut = <<-HTML 109 | 110 | 111 | TITLE 112 | 113 | 114 | 115 | 116 | 

 117 | HTML
 118 | HTMLEnd = <<-HTML
 119 |
120 | 121 | 122 | 123 | HTML 124 | 125 | class User 126 | def initialize(options) 127 | @options = options 128 | @username = options[:user] 129 | @links = {} 130 | @links[:gh] = "https://api.github.com/users/#{@username}" 131 | @links[:gist]= "https://api.github.com/users/#{@username}/gists" 132 | @links[:api] = "https://api.github.com/users/#{@username}/events/public?page=" 133 | @links[:repos] = "https://api.github.com/users/#{@username}/repos?page=" 134 | @info = {} 135 | @page = 1 136 | end 137 | 138 | def stackoverflow(name) 139 | base = "https://stackoverflow.com/users?page=" 140 | filter = "&filter=All&search=" 141 | potential_accounts = [] 142 | search = name ? name : @username 143 | resp = HTTParty.get("#{base}1#{filter}#{search}", headers: { 144 | 'User-Agent' => UserAgent 145 | }).body 146 | # profile link | display name | location | reputation 147 | users = resp.scan(/user-details">\r\n.*?(.*?)<\/a>\r\n.*?(.*?)<\/span>.*?\r\n.*?\r\n.*?dir="ltr">(.*?) user[1], 152 | :link => "https://stackoverflow.com#{user[0]}", 153 | :location => user[2], 154 | :reputation => user[3] 155 | } 156 | if count < 10 157 | # Get additional info from top results 158 | id = user[0].scan(/(\d*)/)[7][0] 159 | profile = HTTParty.get("http://api.stackexchange.com/2.2/users/#{id}?site=stackoverflow&filter=!)RvYQaDu4xmx4JA(JIILy)1X", headers: { 160 | 'User-Agent' => UserAgent 161 | }).body 162 | profile = JSON.parse(profile)["items"][0] 163 | stack_user[:url] = profile["website_url"] 164 | stack_user[:bio] = profile["about_me"] 165 | stack_user[:bio] = stack_user[:bio].gsub(/(<.*?>)/, '').gsub(/\n/, ' ') if stack_user[:bio] 166 | end 167 | potential_accounts.push(stack_user) 168 | count = count + 1 169 | end 170 | potential_accounts 171 | end 172 | 173 | def repos() 174 | repos = [] 175 | page = 1 176 | while true 177 | resp = github_api_req("#{@links[:repos]}#{page}", @options[:auth]) 178 | if resp.empty? or resp == "[]" 179 | break 180 | end 181 | resp = JSON.parse(resp) 182 | check_auth_resp(resp) 183 | resp.each do |repo| 184 | next if repo["fork"] 185 | new = { 186 | :full_name => repo["full_name"], 187 | :repo => repo["name"], 188 | :url => repo["html_url"], 189 | } 190 | repos.push(new) 191 | end 192 | page = page + 1 193 | end 194 | repos 195 | end 196 | 197 | def emails() 198 | emails = [] 199 | while true 200 | resp = github_api_req("#{@links[:api]}#{@page}", @options[:auth]) 201 | resp = JSON.parse(resp) 202 | if resp.empty? 203 | break 204 | end 205 | check_auth_resp(resp) 206 | # It's possible to get the name from here too 207 | resp.each do |x| 208 | next if x["payload"]["commits"] == nil 209 | arr = x["payload"]["commits"] 210 | # Getting the length because for merge commits, the last one 211 | # is the one by our guy actually pushing the external commits 212 | # to the local repo 213 | len = arr.length() - 1 214 | next if len == -1 215 | if @options[:extra_checking] 216 | commit_url = x["payload"]["commits"][len]["url"] 217 | commit = github_api_req(commit_url, @options[:auth]) 218 | commit = JSON.parse(commit) 219 | next if commit["author"] == nil 220 | next if commit["author"]["login"] != @username 221 | end 222 | emails.push(x["payload"]["commits"][len]["author"]["email"]) 223 | end 224 | @page = @page + 1 225 | # Hard limit of 5 pages 226 | if @page > 5 227 | break 228 | end 229 | end 230 | emails.uniq 231 | end 232 | 233 | def gists() 234 | gistlist=[] 235 | resp = github_api_req(@links[:gist], @options[:auth]) 236 | 237 | resp = JSON.parse(resp) 238 | check_auth_resp(resp) 239 | 240 | resp.each do |gist| 241 | new = { 242 | :id =>gist["id"], 243 | :url => gist["html_url"], 244 | } 245 | gistlist.push(new) 246 | end 247 | gistlist 248 | end 249 | 250 | def get_info() 251 | profile = github_api_req(@links[:gh], @options[:auth]) 252 | profile = JSON.parse(profile) 253 | # Special case where check_auth_resp() can't be used 254 | if profile["message"] 255 | STDERR.puts API_ERR_MSG 256 | STDERR.puts profile["message"] 257 | exit 258 | end 259 | display_name = profile["name"] 260 | @info[:display_name] = display_name if display_name 261 | @info[:username] = @username 262 | bio = profile["bio"] 263 | @info[:bio] = bio if bio 264 | works_for = profile["company"] 265 | @info[:works_for] = works_for if works_for 266 | location = profile["location"] 267 | @info[:location] = location if location 268 | public_email = profile["email"] 269 | if public_email 270 | @info[:public_email] = public_email 271 | else 272 | @info[:public_email] = "" 273 | end 274 | public_url = profile["blog"] 275 | @info[:wizard] = "phurd" 276 | @info[:public_url] = public_url if public_url 277 | @info[:repos] = profile["public_repos"] 278 | @info[:gists] = profile["public_gists"] 279 | @info[:followers] = profile["followers"] 280 | @info[:follows] = profile["following"] 281 | return if !profile["organizations_url"] 282 | orgs = github_api_req(profile["organizations_url"], @options[:auth]) 283 | orgs = JSON.parse(orgs) 284 | check_auth_resp(orgs) 285 | organizations = [] 286 | orgs.each do |x| 287 | organizations.push(x["login"]) 288 | end 289 | @info[:organizations] = organizations if organizations 290 | end 291 | 292 | def print_info() 293 | bold = "\033[1;1m" 294 | reg = "\033[0;0m" 295 | printhn bold + @info[:display_name] if @info[:display_name] 296 | printhn "https://github.com/" + @info[:username] + reg 297 | pprinthttn ":bio", @info[:bio] 298 | pprinthtn ":works_for", @info[:works_for] 299 | pprinthtn ":location", @info[:location] 300 | pprinthtn ":public_email", bold + @info[:public_email] + reg 301 | if @options[:loud] 302 | self.url_loud(@info[:public_url]) 303 | else 304 | pprinthtn ":public_url", @info[:public_url] 305 | end 306 | self.whois(@info[:public_url]) if @options[:whois] 307 | pprinthttn ":repos", @info[:repos] 308 | pprinthttn ":gists", @info[:gists] 309 | pprinthtn ":followers", @info[:followers] 310 | pprinthtn ":follows", @info[:follows] 311 | 312 | if @info[:organizations] != nil 313 | printhn "\t:organizations" 314 | @info[:organizations].each do |x| 315 | printhttn x 316 | end 317 | end 318 | @commit_emails = self.emails() 319 | if @commit_emails != nil 320 | printhn "\t:emails" 321 | @commit_emails.each do |x| 322 | if @options[:haveibeenpwned] 323 | self.find_pwned(x, true) 324 | else 325 | printhttn bold + x + reg unless x.match('noreply.github') 326 | end 327 | end 328 | if @options[:haveibeenpwned] and @info[:public_email] and !@commit_emails.include? @info[:public_email] 329 | self.find_pwned(@info[:public_email], true) 330 | end 331 | end 332 | if @options[:stackoverflow] 333 | stackaccs = self.stackoverflow(nil) 334 | if @info[:display_name] and @info[:display_name] != "" and @info[:display_name] != @info[:username] 335 | stackaccs = stackaccs + self.stackoverflow(@info[:display_name]) 336 | end 337 | if stackaccs != nil and stackaccs.length() != 0 338 | printhn "\t:potential stackoverflow accounts" 339 | count = 0 340 | stackaccs.each do |acc| 341 | break if count > 20 342 | stack_decide_bold(acc, @info) 343 | count = count + 1 344 | end 345 | end 346 | end 347 | printhn "" 348 | csv_output().each do |csv| 349 | @options[:csv_accounts].push(csv) 350 | end 351 | end 352 | 353 | def csv_output() 354 | output = [] 355 | firstname = "" 356 | lastname = "" 357 | # Get all the emails 358 | emails = @commit_emails 359 | if @info[:public_email] 360 | emails.push(@info[:public_email]) 361 | end 362 | if emails.length == 0 363 | return 364 | end 365 | # Parse their name into first and last 366 | if @info[:display_name] 367 | name = @info[:display_name].split 368 | firstname = name[0] 369 | if name.length != 1 370 | lastname = name[name.length - 1] 371 | end 372 | end 373 | emails.each do |email| 374 | output.push([firstname, lastname, email, @info[:username]].join(",")) if email.length > 1 and not email =~ /noreply.github/ 375 | end 376 | output 377 | end 378 | 379 | def whois(url) 380 | return if url == nil or url == "" 381 | 382 | # Check whois is installed 383 | a = `which whois 2>&1` 384 | if a.include? "/" 385 | else 386 | STDERR.puts "whois utility not installed.\n" 387 | # Fail 388 | return 389 | end 390 | # Strip .*:// 391 | url = url.sub(/^.*:\/\//, "") 392 | # Strip /.*$ 393 | url = url.sub(/\/.*$/, "") 394 | # Potentially dangerous, but I think GitHub enforces proper URL formatting 395 | a = `mkdir report 2>&1 > /dev/null` 396 | a = `whois #{url} > report/whois_#{url}.txt` 397 | printhttn "report/whois_#{url}.txt created" 398 | end 399 | 400 | # This is a bit of an EyeWitness-type functionality 401 | def url_loud(url) 402 | # Filter out just the domain 403 | domain = url 404 | # thewizard, god of regex 405 | # Strip .*:// 406 | domain = domain.sub(/^.*:\/\//, "") 407 | # Strip /.*$ 408 | domain = domain.sub(/\/.*$/, "") 409 | # Check whether the domain is up 410 | up80 = false 411 | begin 412 | # Make a socket connection 413 | Addrinfo.tcp(domain, 80).connect({ :timeout => 3 }) { |s| 414 | s.print "GET / HTTP/1.1\r\nHost: #{url}\r\n\r\n" 415 | a = s.read 416 | } 417 | up80 = true 418 | rescue => e 419 | # Domain is truly down 420 | up80 = false 421 | end 422 | up443 = false 423 | begin 424 | # Make a socket connection 425 | # This will fail because it's HTTPS but... 426 | sock = TCPSocket.new(domain, 443) 427 | ctx = OpenSSL::SSL::SSLContext.new 428 | ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER) 429 | @socket = OpenSSL::SSL::SSLSocket.new(sock, ctx).tap do |socket| 430 | socket.sync_close = true 431 | socket.connect 432 | end 433 | up443 = true 434 | rescue => e 435 | # Domain is truly down 436 | up443 = false 437 | end 438 | if up80 and up443 439 | pprinthtn(":public_url", "#{url} (up - 80, 443)") 440 | elsif up80 441 | pprinthtn(":public_url", "#{url} (up - 80)") 442 | elsif up443 443 | pprinthtn(":public_url", "#{url} (up - 443)") 444 | else 445 | pprinthtn(":public_url", "#{url} (down)") 446 | return 447 | end 448 | # Firefox screenshots disabled 449 | # Screenshot 450 | #while `ps aux | grep firefox | grep -v grep`.include? "firefox" 451 | # STDERR.puts "Please close Firefox and press enter to allow screenshots\n" 452 | # a = STDIN.gets 453 | #end 454 | a = `mkdir report 2>&1 > /dev/null` 455 | #a = `firefox -no-remote -screenshot #{@username}_#{domain}.png #{url} 2>&1 > /dev/null && sleep .5 && mv #{@username}_#{domain}.png report/#{@username}_#{domain}.png` 456 | #printhttn("report/#{@username}_#{domain}.png created") 457 | begin 458 | page = HTTParty.get( url, headers: { 459 | 'User-Agent' => '', 460 | } ) 461 | rescue => e 462 | STDERR.puts "Network error:\n#{e}\n" 463 | return 464 | end 465 | File.write("report/#{@username}_url.html", page) 466 | printhttn "report/#{@username}_url.html created" 467 | if @options[:wl] 468 | @options[:pwHash].update(frequency_hash(page)) 469 | end 470 | end 471 | 472 | def find_pwned(email, tobold) 473 | sleep(1.5) # Required for haveibeenpwned rate limiting 474 | # Should parse these out earlier 475 | return if email.match('noreply.github') 476 | return unless email.include? "@" 477 | api = "https://haveibeenpwned.com/api/v2/breachedaccount/" 478 | truncate = "?truncateResponse=true" 479 | bold = "\033[1;1m" 480 | reg = "\033[0;0m" 481 | begin 482 | breaches = HTTParty.get( "#{api}#{email}", headers: { 483 | 'User-Agent' => UserAgent, 484 | } ) 485 | rescue => e 486 | STDERR.puts "Network error:\n#{e}\n" 487 | return 488 | end 489 | if tobold 490 | printhttn bold + email + reg 491 | else 492 | printhttn email 493 | end 494 | return if breaches.code != 200 495 | return if breaches.body.include? "Page not found" 496 | breaches = JSON.parse(breaches.body) 497 | return unless breaches and breaches.length() != 0 498 | breaches.each do |breach| 499 | printhttn "\tBreached in #{breach["BreachDate"][0..3]}: #{breach["Name"]}" 500 | end 501 | end 502 | 503 | def stack_decide_bold(stack, git) 504 | bold = "\033[1;1m" 505 | reg = "\033[0;0m" 506 | tobold = false 507 | if stack[:location] and stack[:location] != "" and stack[:location] == git[:location] 508 | tobold = true 509 | elsif stack[:url] and stack[:url] != "" and stack[:url][/:\/\/(.*?)\/?$/, 0] == git[:public_url][/:\/\/(.*?)\/?$/, 0] 510 | tobold = true 511 | end 512 | printh bold if tobold 513 | printttn stack[:link] 514 | line2 = "\t" 515 | line2 = "\t#{stack[:location]} | " if stack[:location] != "" 516 | line2 = "#{line2}#{stack[:reputation]} rep" 517 | line2 = "#{line2} | #{stack[:url]}" if stack[:url] and stack[:url] != "" 518 | printhttn line2 519 | printhttn "\t#{stack[:bio]}" if stack[:bio] != "" if tobold 520 | printh reg if tobold 521 | end 522 | 523 | def bio() 524 | return @info[:bio] if @info[:bio] 525 | end 526 | end 527 | 528 | class Organization 529 | def initialize(org, auth) 530 | @org = org 531 | @page = 1 532 | @peopleurl = "https://api.github.com/orgs/#{org}/members?page=" 533 | @reposurl = "https://api.github.com/orgs/#{org}/repos?page=" 534 | @auth = auth 535 | end 536 | 537 | def people() 538 | people = [] 539 | while true 540 | new = github_api_req("#{@peopleurl}#{@page}", @auth) 541 | if new.empty? 542 | break 543 | end 544 | new = JSON.parse(new) 545 | check_auth_resp(new) 546 | new.each do |x| 547 | people.push(x["login"]) 548 | end 549 | @page = @page + 1 550 | break if @page > 5 551 | end 552 | people 553 | end 554 | 555 | def repos() 556 | repos = [] 557 | page = 1 558 | while true 559 | resp = github_api_req("#{@reposurl}#{page}", @auth) 560 | if resp.empty? or resp == "[]" 561 | break 562 | end 563 | resp = JSON.parse(resp) 564 | check_auth_resp(resp) 565 | resp.each do |repo| 566 | next if repo["fork"] 567 | new = { 568 | :full_name => repo["full_name"], 569 | :repo => repo["name"], 570 | :url => repo["html_url"], 571 | } 572 | repos.push(new) 573 | end 574 | page = page + 1 575 | end 576 | repos 577 | end 578 | end 579 | 580 | class Gist 581 | def initialize(owner, gist_id, auth) 582 | @owner= owner 583 | @gist_id= gist_id 584 | @auth= auth 585 | end 586 | 587 | def url() 588 | "https://github.com/gists/#{@gist_id}" 589 | end 590 | 591 | end 592 | 593 | class Repo 594 | def initialize(owner, repo, auth) 595 | @owner = owner 596 | @repo = repo 597 | @auth = auth 598 | @page = 1 599 | @links = { 600 | :link => "https://api.github.com/repos/#{@owner}/#{@repo}", 601 | :contributors => "https://api.github.com/repos/#{@owner}/#{@repo}/contributors?page=", 602 | :readme => "https://api.github.com/repos/#{@owner}/#{@repo}/readme" 603 | } 604 | end 605 | 606 | def contributors() 607 | contributors = [] 608 | while true 609 | resp = github_api_req("#{@links[:contributors]}#{@page}", @auth) 610 | new = JSON.parse(resp) 611 | if new.empty? 612 | break 613 | end 614 | check_auth_resp(new) 615 | new.each do |cont| 616 | contributors.push(cont["login"]) 617 | end 618 | @page = @page + 1 619 | end 620 | contributors 621 | end 622 | 623 | def url() 624 | "https://github.com/#{@owner}/#{@repo}" 625 | end 626 | 627 | def wordlist(pwHash) 628 | printhn "Gathering words from #{@owner}/#{@repo}" 629 | resp = github_api_req(@links[:readme], @auth) 630 | resp = JSON.parse(resp) 631 | 632 | if !resp["message"].eql? "Not Found" # if there is a readme available 633 | readme = Base64.decode64(resp["content"]) 634 | pwHash.update(frequency_hash(readme)) # update global password list 635 | end 636 | 637 | resp = github_api_req(@links[:link], @auth) 638 | resp = JSON.parse(resp) 639 | description = resp["description"] 640 | if description != nil 641 | pwHash.update(frequency_hash(description)) 642 | end 643 | end 644 | end 645 | 646 | class Local 647 | def initialize(options) 648 | @options = options 649 | @committers = [] 650 | @authors = [] 651 | # TODO: cd and run git log on the path to make sure it's initialized 652 | end 653 | 654 | def scrape() 655 | # git log --pretty=format:"%an|%ae|%cn|%ce" 656 | printhn "Authors:" 657 | printhn `cd "#{@options[:local]}" && git log --pretty=format:"%an - %ae" | sort -u` 658 | printhn "Committers:" 659 | printhn `cd "#{@options[:local]}" && git log --pretty=format:"%cn - %ce" | sort -u` 660 | `cd "#{@options[:local]}" && git log --pretty=format:"%an|%ae|%cn|%ce"`.split(/\r?\n/).each do |line| 661 | format = line.split(/\|/) 662 | @authors.push({:name => format[0], :email => format[1]}) 663 | @committers.push({:name => format[2], :email => format[3]}) 664 | end 665 | @committers = @committers.uniq 666 | @authors = @authors.uniq 667 | csv_output().each do |csv| 668 | @options[:csv_accounts].push(csv) 669 | end 670 | end 671 | 672 | def csv_output() 673 | output = [] 674 | @committers.each do |c| 675 | firstname = "" 676 | lastname = "" 677 | position = "" 678 | name = c[:name].split 679 | firstname = name[0] 680 | if name.length != 1 681 | lastname = name[name.length - 1] 682 | end 683 | output.push([firstname, lastname, c[:email], position].join(",")) 684 | end 685 | @authors.each do |a| 686 | firstname = "" 687 | lastname = "" 688 | position = "" 689 | name = a[:name].split(" ") 690 | firstname = name[0] 691 | if name.length != 1 692 | lastname = name[name.length - 1] 693 | end 694 | output.push([firstname, lastname, a[:email], position].join(",")) 695 | end 696 | output.uniq 697 | end 698 | 699 | def mine() 700 | regex = "PRIVATE KEY|A[A-Z]IA[A-Z]{8,}|[Pp][Aa][Ss]{2}[Ww][Oo]?[Rr]?[Dd].{,2}\\s*?[=:]|s3\.amazonaws\.com/|secret_key_base\\s*?[=:]" 701 | printn "Debug: creating report dir" 702 | a = `mkdir report 2>&1 > /dev/null` 703 | printn "Debug: creating mine.temp" 704 | a = `touch mine.temp` 705 | # Search through the repo 706 | currentlocation = `pwd`.rstrip 707 | printn "Debug: running git log command" 708 | printn "cd #{@options[:local]} && git log --pickaxe-regex -p --color-words -S #{regex} \":(exclude)*jquery*\" > #{currentlocation}/mine.temp && cd #{currentlocation}" 709 | `cd "#{@options[:local]}" && git log --pickaxe-regex -p --color-words -S "#{regex}" ":(exclude)*jquery*"> "#{currentlocation}/mine.temp" && cd "#{currentlocation}"` 710 | # Grab only the commit numbers, fine names, and the terms we're searching for 711 | filename = "" # Filename 712 | line_num = 0 713 | printn "Debug: creating mined.temp" 714 | a = `touch mined.temp` 715 | a = `printf "Generated with git-user.rb, created by Patrick Hurd @ Coalfire Federal\n\n" > mined.temp` 716 | results = 0 717 | 718 | if( File.size("mine.temp") > 0 ) #does this file have any results 719 | File.readlines("mine.temp").each do |line| 720 | if ( line =~ /commit/ ) 721 | File.write("mined.temp", line, File.size("mined.temp"), mode: "a") 722 | elsif ( line =~ /diff --git a/ ) 723 | filename = line.rstrip 724 | elsif ( line =~ /@@ .*? @@/ ) 725 | line_num = line[/\s\+(\d+),/, 0].to_i 726 | elsif ( line =~ /#{regex}/ ) # we found a match 727 | File.write("mined.temp", "#{filename[15..-1]}:#{line_num} #{line.strip}\n", File.size("mined.temp"), mode: "a") 728 | results += 1 729 | line_num += 1 730 | else 731 | line_num += 1 732 | end 733 | end 734 | else 735 | return 736 | end 737 | printh "Writing ./report/#{@options[:name]}_mined.html (results: #{results})\n" 738 | a = `cat mined.temp | aha -t #{@options[:name]} > "report/#{@options[:name]}_mined.html"` 739 | return 740 | # TODO: Figure out whether there's an upstream URL for the local repo and reference that 741 | # Post-processing 742 | betterhtml = "" 743 | File.readlines("report/#{@options[:name]}_mined.html").each do |line| 744 | if ( line =~ /olive;">commit / ) 745 | commit = /commit ([0-9a-f]*)/.match(line).captures 746 | commit = commit[0] 747 | if ( repo_url =~ /gist.github.com/ ) 748 | betterhtml += "commit #{commit}\n" 749 | else 750 | betterhtml += "commit #{commit}\n" 751 | end 752 | else 753 | betterhtml += line 754 | end 755 | end 756 | File.write("report/#{repo}_mined.html", betterhtml) 757 | a = `rm mine.temp mined.temp` 758 | end 759 | end 760 | 761 | def frequency_hash(string) 762 | words = string.split(' ') 763 | frequency = Hash.new(0) 764 | words.each { |word| frequency[word.downcase] += 1 } 765 | frequency.delete_if { |key, value| value < 2 } # if the word is used less than 2 times, delete it 766 | frequency.delete_if { |key, value| key.length < 4 }# if the word is shorter than 4 characters, delete it 767 | frequency.each do |key,value| 768 | if key =~ /^[a-zA-Z0-9]*$/ 769 | #STDERR.puts "KEY: #{key} matches" 770 | else 771 | frequency.delete(key) 772 | #STDERR.puts "KEY: #{key} deleted." 773 | end 774 | end 775 | frequency 776 | end 777 | 778 | def mine_repo(repo_url) 779 | regex = "PRIVATE KEY|A[A-Z]IA[A-Z]{8,}|[Pp][Aa][Ss]{2}[Ww][Oo]?[Rr]?[Dd].{,2}\\s*?[=:]|s3\.amazonaws\.com/|secret_key_base\\s*?[=:]" 780 | repo = repo_url.scan(/\/([A-Za-z0-9\-_\.]*)$/)[0][0] 781 | 782 | a = `mkdir report 2>&1 > /dev/null` 783 | # Clone repo 784 | a = `git clone #{repo_url} 2>&1` 785 | a = `touch mine.temp` 786 | # Search through the repo 787 | `cd #{repo} && git log --pickaxe-regex -p --color-words -S "#{regex}" ":(exclude)*jquery*" > ../mine.temp && cd .. && rm -rf #{repo}` 788 | # Grab only the commit numbers, fine names, and the terms we're searching for 789 | filename = "" # Filename 790 | line_num = 0 791 | a = `touch mined.temp` 792 | a = `printf "Generated with git-user.rb, created by Patrick Hurd @ Coalfire Federal\n\n" > mined.temp` 793 | results=0 794 | 795 | if(File.size("mine.temp")>0) #does this file have any results 796 | File.readlines("mine.temp").each do |line| 797 | if ( line =~ /commit/ ) 798 | File.write('mined.temp', line, File.size('mined.temp'), mode: 'a') 799 | #a = `echo "#{line}" >> mined.temp` 800 | elsif ( line =~ /diff --git a/ ) 801 | filename = line.rstrip 802 | elsif ( line =~ /@@ .*? @@/ ) 803 | line_num = line[/\s\+(\d+),/, 0].to_i 804 | elsif ( line =~ /#{regex}/ ) # we found a match 805 | File.write('mined.temp', "#{filename[15..-1]}:#{line_num} #{line.strip}\n", File.size('mined.temp'), mode: 'a') 806 | #a = `echo "#{filename[15..-1]}:#{line_num} #{line.strip}" >> mined.temp` 807 | results+=1 808 | line_num += 1 809 | else 810 | line_num += 1 811 | end 812 | end 813 | printh "Writing ./report/#{repo}_mined.html (results: #{results})\n" 814 | a = `cat mined.temp | aha -t #{repo} > report/#{repo}_mined.html` 815 | # Post-processing 816 | betterhtml = "" 817 | File.readlines("report/#{repo}_mined.html").each do |line| 818 | if ( line =~ /olive;">commit / ) 819 | commit = /commit ([0-9a-f]*)/.match(line).captures 820 | commit = commit[0] 821 | if ( repo_url =~ /gist.github.com/ ) 822 | betterhtml += "commit #{commit}\n" 823 | else 824 | betterhtml += "commit #{commit}\n" 825 | end 826 | else 827 | betterhtml += line 828 | end 829 | end 830 | File.write("report/#{repo}_mined.html", betterhtml) 831 | end 832 | a = `rm mine.temp mined.temp` 833 | end 834 | 835 | options = { :pwHash => pwHash } 836 | 837 | OptionParser.new do |parser| 838 | parser.banner = "Usage: git-user.rb [options]" 839 | 840 | parser.on("-h", "--help", "Show this help banner") do || 841 | puts parser 842 | print "\n" 843 | print "Tip: --repo needs either -o or -u to be set\n\n" 844 | print "Tip: --extra_checks needs -a or -t to make authenticated API calls\n\n" 845 | print "Tip: --pwned and --csv needs -e to be set to ensure your scope is correct\n\n" 846 | print "Created by Patrick Hurd @ Coalfire Federal\n" 847 | exit 848 | end 849 | 850 | # Targetting options 851 | parser.on("-u", "--user USERNAME", "User to gather info from") do |v| 852 | options[:user] = v 853 | end 854 | parser.on("-o", "--organization ORGANIZATION", "Organization to scrape") do |v| 855 | options[:org] = v 856 | end 857 | parser.on("-r", "--repo REPO", "The repo whom's contributors to scrape") do |v| 858 | options[:repo] = v 859 | end 860 | parser.on("--local ABSOLUTE_PATH", "Perform scrape on a repo local to your filesystem") do |v| 861 | options[:local] = v 862 | end 863 | parser.on("--name NAME", "Name to refer to a --local repo in report filenames") do |v| 864 | options[:name] = v 865 | end 866 | 867 | # Authentication options 868 | parser.on("-a", "--auth", "Authenticate with HTTP basic auth") do || 869 | options[:auth] = true 870 | end 871 | parser.on("-t", "--token TOKEN", "Use specified GitHub personal access token") do |v| 872 | options[:token] = v 873 | end 874 | 875 | # Scraper activities 876 | parser.on("-s", "--stackoverflow", "Try to find users' accounts on StackOverflow") do || 877 | options[:stackoverflow] = true 878 | end 879 | parser.on("-p", "--pwned", "Search for relevant data breaches using haveibeenpwned") do || 880 | options[:haveibeenpwned] = true 881 | end 882 | parser.on("-e", "--extra_checking", "Do extra checking on email addresses") do || 883 | options[:extra_checking] = true 884 | end 885 | parser.on("-m", "--mine", "Mine the repo or user/organization's repos for secrets") do || 886 | options[:mine] = true 887 | end 888 | parser.on("--whois", "Perform whois lookup on domains found in profile information") do || 889 | options[:whois] = true 890 | end 891 | parser.on("-l", "--loud", "Perform active recon on users (scrape their personal site)") do || 892 | options[:loud] = true 893 | end 894 | 895 | # Output options 896 | parser.on("--html", "Output main report to an HTML document") do || 897 | options[:html] = true 898 | end 899 | parser.on("-w", "--wordlist", "Generate wordlist for use in password attacks") do || 900 | options[:wl] = true 901 | end 902 | parser.on("-c", "--csv", "Export discovered accounts to a GoPhish-importable CSV file") do || 903 | options[:csv] = true 904 | end 905 | end.parse! 906 | 907 | logo = <<-LOGO 908 | ___ ___ ___ ___ ___ ___ ___ 909 | /\\ \\ /\\ \\ /\\ \\ /\\__\\ /\\ \\ /\\ \\ /\\ \\ 910 | / \\ \\ _\\ \\ \\ \\ \\ \\ / / _/_ / \\ \\ / \\ \\ / \\ \\ 911 | / /\\ \\__\\ /\\/ \\__\\ / \\__\\ / /_/\\__\\ /\\ \\ \\__\\ / \\ \\__\\ / \\ \\__\\ 912 | \\ \\ \\/__/ \\ /\\/__/ / /\\/__/ \\ \\/ / / \\ \\ \\/__/ \\ \\ \\/ / \\, / / 913 | \\ / / \\ \\__\\ \\/__/ \\ / / \\ / / \\ \\/ / | \\/__/ 914 | \\/__/ \\/__/ \\/__/ \\/__/ \\/__/ \\|__| 915 | 916 | Created by Patrick Hurd @ Coalfire Federal 917 | LOGO 918 | 919 | options[:csv_accounts] = [] 920 | if options[:csv] and !options[:extra_checking] and !options[:local] 921 | print "Extra checking must be enabled with -e to use -c.\n" 922 | print "This helps ensure you do not get incorrect results..\n\n" 923 | exit 924 | end 925 | if options[:haveibeenpwned] and !options[:extra_checking] 926 | print "Extra checking must be enabled with -e to use -p.\n" 927 | print "This ensures you do not reach beyond the intended scope.\n\n" 928 | exit 929 | end 930 | if options[:extra_checking] and !(options[:auth] or options[:token]) 931 | print "GitHub basic authentication must be enabled with -a to use -e." 932 | print "Extra checking will chew through your unauthenticated API limit.\n\n" 933 | exit 934 | end 935 | if options[:mine] and !`which aha`.include? "/aha" 936 | print "git-user.rb uses aha to generate mine reports." 937 | print "Install aha with:" 938 | print "sudo apt install aha -y" 939 | exit 940 | end 941 | 942 | if options[:auth] 943 | print "Using HTTP basic auth\n" 944 | print "Enter your username: " 945 | username = gets.chomp 946 | print "Enter your password: " 947 | password = STDIN.noecho(&:gets).chomp 948 | print "\n" 949 | options[:auth] = { :username => username, :password => password } 950 | elsif options[:token] 951 | options[:auth] = { :token => options[:token] } 952 | else 953 | options[:auth] = { :username => "", :password => "" } 954 | end 955 | 956 | if options[:repo] 957 | owner = options[:org] ? options[:org] : options[:user] 958 | if owner == nil 959 | print "\nYou need to specify the user or organization who owns the repo.\n\n" 960 | exit 961 | end 962 | printh logo + "\ngit-user.rb report for https://github.com/" + owner + "/" + options[:repo] + "\n\n" 963 | repo = Repo.new(owner, options[:repo], options[:auth]) 964 | repo.contributors().each do |person| 965 | options[:user] = person 966 | user = User.new(options) 967 | user.get_info() 968 | user.print_info() 969 | end 970 | if options[:mine] 971 | printhn "Mining #{owner}/#{options[:repo]}" 972 | STDERR.puts "Warning: terminating git-user.rb while mining may leave data fragments in your working directory." 973 | mine_repo(repo.url()) 974 | 975 | end 976 | if options[:wl] 977 | repo.wordlist(pwHash) 978 | end 979 | elsif options[:user] 980 | printh logo + "\n" 981 | user = User.new(options) 982 | user.get_info() 983 | user.print_info() 984 | if options[:mine] 985 | STDERR.puts "Warning: terminating git-user.rb while mining may leave data fragments in your working directory." 986 | user.repos().each do |repo| 987 | printhn "Mining #{options[:user]}/#{repo[:repo]}" 988 | mine_repo(repo[:url]) 989 | 990 | end 991 | user.gists().each do |gist| 992 | printhn "Mining #{options[:user]}/#{gist[:id]}" 993 | mine_repo(gist[:url]) 994 | end 995 | end 996 | if options[:wl] 997 | printhn "Generating Password Wordlist\n" 998 | user.repos().each do |repo| 999 | repo = Repo.new(options[:user], repo[:repo], options[:auth]) 1000 | repo.wordlist(pwHash) 1001 | end 1002 | pwHash.update(frequency_hash(user.bio())) 1003 | end 1004 | elsif options[:org] 1005 | printh logo + "\ngit-user.rb report for https://github.com/" + options[:org] + "\n\n" 1006 | org = Organization.new(options[:org], options[:auth]) 1007 | org.people().each do |person| 1008 | options[:user] = person 1009 | user = User.new(options) 1010 | user.get_info() 1011 | user.print_info() 1012 | end 1013 | if options[:mine] 1014 | STDERR.puts "Warning: terminating git-user.rb while mining may leave data fragments in your working directory." 1015 | org.repos().each do |repo| 1016 | printhn "Mining #{options[:org]}/#{repo[:repo]}" 1017 | mine_repo(repo[:url]) 1018 | end 1019 | end 1020 | if options[:wl] 1021 | printhn "\nGenerating Password Wordlist\n\n" 1022 | org.repos().each do |repo| 1023 | repo = Repo.new(options[:user], repo[:repo], options[:auth]) 1024 | repo.wordlist(pwHash) 1025 | end 1026 | end 1027 | elsif options[:local] 1028 | printh logo + "\ngit-user.rb report for " + options[:local] + " - " + options[:name] + "\n\n" 1029 | if not options[:name] 1030 | STDERR.puts "A --name is required for --local repos" 1031 | end 1032 | l = Local.new(options) 1033 | l.scrape() 1034 | if options[:mine] 1035 | l.mine() 1036 | end 1037 | else 1038 | print `ruby #{$0} --help` 1039 | end 1040 | 1041 | if options[:haveibeenpwned] 1042 | printh "Breached account data courtesy of https://haveibeenpwned.com\n" 1043 | end 1044 | 1045 | if options[:html] 1046 | HTMLOut << HTMLEnd 1047 | name = options[:name] ? options[:name] : options[:org] ? options[:org] : options[:user] 1048 | File.write("./report_#{name}.html", HTMLOut) 1049 | end 1050 | 1051 | if options[:wl] 1052 | pwHash=pwHash.sort {|x,y| x[1]<=>y[1]} 1053 | 1054 | pwHash = Hash[pwHash.reverse] 1055 | a = `mkdir report 2>&1 > /dev/null` 1056 | File.open("./report/wordlist.txt", "w") do |file| 1057 | pwHash.keys.each do |key| 1058 | file.write("#{key}\n") 1059 | end 1060 | printhn("Generated wordlist ./report/wordlist.txt") 1061 | end 1062 | end 1063 | 1064 | if options[:csv] 1065 | name = options[:name] ? options[:name] : options[:org] ? options[:org] : options[:user] 1066 | a = `mkdir report 2>&1 > /dev/null` 1067 | File.open("./report/#{name}_gophish.csv", "w") do |file| 1068 | file.write("First Name,Last Name,Email,Position\n") 1069 | options[:csv_accounts].each do |account| 1070 | file.write("#{account}\n") 1071 | end 1072 | printhn("Generated GoPhish CSV ./report/#{name}_gophish.csv") 1073 | end 1074 | end 1075 | --------------------------------------------------------------------------------