├── .gitignore ├── Crate.py ├── GitLoot.py ├── Map.py ├── MetalDetector.py ├── Prospector.py ├── README.md ├── Shovel.py ├── clues ├── contents.json ├── extensions.json ├── filenames.json ├── messages.json └── paths.json ├── crates ├── FileCrate.py └── SQLCrate.py ├── maps ├── BitbucketMap.py ├── GithubMap.py └── __init__.py ├── models ├── Commit.py ├── Job.py ├── Loot.py ├── Organization.py ├── Repository.py ├── User.py └── __init__.py ├── requirements.txt ├── setup.py └── shovels ├── GitShovel.py └── __init__.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | workbench/ 3 | -------------------------------------------------------------------------------- /Crate.py: -------------------------------------------------------------------------------- 1 | # Handle data storage of a given loot object. Maybe a database, maybe files, whatever we want. 2 | import crates 3 | class Crate: 4 | 5 | def __init__(self, crateType): 6 | #if crateType.lower() == "file": 7 | # self._crate = crates.FileCrate() 8 | if crateType.lower() == "sql": 9 | self._crate = crates.SQLCrate() 10 | else: 11 | return -1 12 | 13 | def addUser(self, user): 14 | self._crate.addUser(user) 15 | return 0 16 | 17 | def addOrganization(self, org): 18 | self._crate.addOrganization(org) 19 | return 0 20 | 21 | def addRepository(self, repo): 22 | self._crate.addRepository(repo) 23 | return 0 24 | 25 | def addLoot(self, loot): 26 | self._crate.addLoot(loot) 27 | return 0 28 | 29 | def addJob(self,job): 30 | self._crate.addJob(job) 31 | return 32 | 33 | def finishJob(self,job): 34 | self._crate.finishJob(job) 35 | return -------------------------------------------------------------------------------- /GitLoot.py: -------------------------------------------------------------------------------- 1 | from ConfigParser import ConfigParser 2 | import argparse 3 | import Map 4 | import Shovel 5 | import MetalDetector 6 | from setup import Setup 7 | import os 8 | import git 9 | #from models import * 10 | 11 | PROG_NAME = "GitLoot" 12 | PROG_VER = 0.1 13 | PROG_DESC = "Automatically perform analysis on repositories to look for valuable information." 14 | PROG_EPILOG = "You know how to do it now, go forth and loot and pillage." 15 | DEPTH = 1 16 | 17 | def parseArgs(): 18 | parser = argparse.ArgumentParser(prog=PROG_NAME, description=PROG_DESC, epilog=PROG_EPILOG) 19 | parser.add_argument("--version", action="version", version="%(prog)s v"+str(PROG_VER)) 20 | # parser.add_argument("--api", nargs='?', dest="job_api", help="The root of the api you want to use for the input job. This overrides the default set in the config.") 21 | # parser.add_argument("--keys", nargs='?', dest="job_keys", help="An (optionally) comma separated list of api keys. A single key is fine for small jobs.") 22 | parser.add_argument("--depth", dest="depth", default=1, help="Scan depth [Default 1]") 23 | parser.add_argument("-o", "--org", dest="org", help="Scan an organization's repositories") 24 | parser.add_argument("-r", "--repository", dest="repo", help="Scan a single repository. Use the full name. E.g. '0xdade/GitLoot'") 25 | parser.add_argument("-u", "--user", dest="user", help="Scan a user's repositories.") 26 | parser.add_argument("-m", "--members", action="store_true", default=False, dest="members", help="Search the repositories of every member of an organization.") 27 | args = parser.parse_args() 28 | if not args.org and not args.repo and not args.user: 29 | print "You must supply either -o, -r, or -u." 30 | raise SystemExit(-1) 31 | if args.org and (args.repo or args.user): 32 | print "Please supply only one target with -o, -r, or -u." 33 | raise SystemExit(-1) 34 | if args.repo and (args.org or args.user): 35 | print "Please supply only one target with -o, -r, or -u." 36 | raise SystemExit(-1) 37 | if args.user and (args.org or args.repo): 38 | print "Please supply only one target with -o, -r, or -u." 39 | raise SystemExit(-1) 40 | if args.repo and '/' not in args.repo: 41 | print "Please supply the full name of the repo. E.g. '0xdade/GitLoot'" 42 | raise SystemExit(-1) 43 | DEPTH = args.depth 44 | return args 45 | 46 | def scanRepo(mmap,shovel,detector,target): 47 | repo = mmap.getRepository(target) 48 | shovel.setRepo(repo) 49 | shovel.clone() 50 | commitCount = 0 51 | for commit in shovel.nextCommit(): 52 | if commitCount < DEPTH: 53 | cluesFound = detector.processCommit(commit) 54 | commitCount+=1 55 | else: 56 | print "Clues Found: " + str(cluesFound) 57 | break 58 | shovel.cleanUp() 59 | 60 | def scanUser(mmap,shovel,detector,target): 61 | repos = target.getRepos() 62 | for rid,full_name in repos: 63 | scanRepo(mmap,shovel,detector,full_name) 64 | 65 | def scanOrg(mmap,shovel,detector,target,members): 66 | org = mmap.getOrganization(target) 67 | repos = target.getRepos() 68 | for rid,full_name in repos: 69 | scanRepo(mmap,shovel,detector,full_name) 70 | if members: 71 | for uid,login in org.getMembers(): 72 | user = mmap.getUser(login) 73 | scanUser(mmap,shovel,detector,user) 74 | 75 | 76 | def main(): 77 | if not (os.path.isfile(os.path.expanduser('~') + "/.gitloot")): 78 | # config not found, we need to prompt the user for config settings 79 | setup = Setup() 80 | print "Setup completed successfully. Please re-run GitLoot to begin." 81 | raise SystemExit(0) 82 | # We load the config first, and then read command line options. Command line can override options from the config at runtime. 83 | args = parseArgs() 84 | 85 | gh = Map.Map('github') 86 | shovel = Shovel.Shovel('git') 87 | md = MetalDetector.MetalDetector() 88 | 89 | if args.org: 90 | scanOrg(gh, shovel, md, args.org, args.members) 91 | elif args.user: 92 | scanUser(gh, shovel, md, args.user) 93 | elif args.repo: 94 | scanRepo(gh, shovel, md, args.repo) 95 | 96 | #myRepo = gh.getRepository('0xdade/GitLoot') 97 | #print str(org) + "\n" 98 | # for rid,full_name in org.getRepos(): 99 | # repo = gh.getRepository(full_name) 100 | # shovel.setRepo(repo) 101 | # shovel.clone() 102 | # shovel.cleanUp() 103 | # print str(repo) + "\n" 104 | 105 | # for uid,login in org.getMembers(): 106 | # user = gh.getUser(login) 107 | # # print str(user) + "\n" 108 | # for rid,full_name in user.getRepos(): 109 | # repo = gh.getRepository(full_name) 110 | # shovel.setRepo(repo) 111 | # shovel.clone() 112 | # print "\n" + str(repo) 113 | # commitCount = 0 114 | # for commit in shovel.nextCommit(): 115 | # if commitCount < args.depth: 116 | # cluesFound = md.processCommit(commit) 117 | # commitCount+=1 118 | # else: 119 | # print "Clues Found: " + str(cluesFound) 120 | # break 121 | 122 | # shovel.cleanUp() 123 | # #gh.getUsers() 124 | # #print "I guess we're done here. . ." 125 | 126 | 127 | if __name__ == '__main__': 128 | main() 129 | -------------------------------------------------------------------------------- /Map.py: -------------------------------------------------------------------------------- 1 | class Map: 2 | 3 | def __init__(self, mapType): 4 | import maps 5 | if mapType.lower() == "bitbucket": 6 | self._map = maps.BitbucketMap() 7 | elif mapType.lower() == "github": 8 | self._map = maps.GithubMap() 9 | 10 | def getUser(self, user): 11 | return self._map.getUser(user) 12 | 13 | def getUsers(self): 14 | return self._map.getUsers() 15 | 16 | def getOrganization(self, orgName): 17 | return self._map.getOrganization(orgName) 18 | 19 | def getRepository(self, full_name): 20 | return self._map.getRepository(full_name) -------------------------------------------------------------------------------- /MetalDetector.py: -------------------------------------------------------------------------------- 1 | # Pass in commit objects, study the blobs, attempt to match on clues specified in ./clues/*.json 2 | # When clue is found, create new Loot and pass to crate 3 | import glob, git, json, re 4 | class MetalDetector: 5 | def __init__(self): 6 | self.contents = json.load(open('clues/contents.json')) 7 | self.extensions = json.load(open('clues/extensions.json')) 8 | self.filenames = json.load(open('clues/filenames.json')) 9 | self.messages = json.load(open('clues/messages.json')) 10 | self.paths = json.load(open('clues/paths.json')) 11 | print "MetalDetector: loaded %s clues" % (len(self.contents) + len(self.extensions) + len(self.filenames) + len(self.messages) + len(self.paths)) 12 | 13 | def scanContents(self, content): 14 | clueCounter = 0 15 | for clue in self.contents: 16 | try: 17 | if clue['type'] == "match": 18 | for line in content.split('\n'): 19 | if clue['pattern'] in line: 20 | print "CONTENT MATCH CLUE: " + line 21 | clueCounter += 1 22 | if clue['type'] == "regex": 23 | match = re.search(clue['pattern'], content) 24 | if match != None: 25 | print "CONTENT REGEX CLUE: " + str(match.groups()) 26 | clueCounter += 1 27 | except UnicodeDecodeError: 28 | pass 29 | return clueCounter 30 | 31 | def scanExtension(self, extension): 32 | clueCounter = 0 33 | for clue in self.extensions: 34 | try: 35 | if clue['type'] == "match" and clue['pattern'] == extension: 36 | print "EXTENSION MATCH CLUE: " + extension 37 | clueCounter += 1 38 | if clue['type'] == "regex" and (re.search(clue['pattern'], extension) != None): 39 | print "EXTENSION MATCH CLUE: " + extension 40 | clueCounter += 1 41 | except UnicodeDecodeError: 42 | pass 43 | return clueCounter 44 | 45 | def scanFilename(self, filename): 46 | clueCounter = 0 47 | for clue in self.filenames: 48 | try: 49 | if clue['type'] == "match" and clue['pattern'] == filename: 50 | print "FILENAME MATCH CLUE: " + filename 51 | clueCounter += 1 52 | if clue['type'] == "regex" and (re.search(clue['pattern'], filename) != None): 53 | print "FILENAME REGEX CLUE: " + filename 54 | clueCounter += 1 55 | except UnicodeDecodeError: 56 | pass 57 | return clueCounter 58 | 59 | def scanMessage(self, message): 60 | clueCounter = 0 61 | for clue in self.messages: 62 | try: 63 | if clue['pattern'] in message: 64 | print "MESSAGE MATCH CLUE: " + message 65 | clueCounter += 1 66 | except UnicodeDecodeError: 67 | pass 68 | return clueCounter 69 | 70 | def scanPath(self, path): 71 | clueCounter = 0 72 | for clue in self.paths: 73 | try: 74 | if clue['type'] == "match" and clue['pattern'] == path: 75 | print "PATH MATCH CLUE: " + path 76 | clueCounter += 1 77 | if clue['type'] == "regex" and (re.search(clue['pattern'], path) != None): 78 | print "PATH REGEX CLUE: " + path 79 | clueCounter += 1 80 | except UnicodeDecodeError: 81 | pass 82 | return clueCounter 83 | 84 | def processTree(self, tree, count): 85 | count = count 86 | clueCount = 0 87 | for item in tree: 88 | if type(item) is git.objects.tree.Tree: 89 | clueCount += self.processTree(item, count+1) 90 | if type(item) is git.objects.blob.Blob: 91 | #print "-"*count + "B: " + str(item) 92 | clueCount += self.scanPath(item.path) 93 | clueCount += self.scanFilename(item.name) 94 | if len(item.name.rsplit('.', 1)) > 1: 95 | clueCount += self.scanExtension(item.name.rsplit('.', 1)[1]) 96 | clueCount += self.scanContents(item.data_stream.read()) 97 | return clueCount 98 | 99 | def processCommit(self, commit): 100 | #print "#"*10 + "\nC: " + str(commit) + "\n" + "#"*10 101 | clueCount = 0 102 | clueCount += self.scanMessage(commit.message) 103 | #print commit.author.email 104 | clueCount += self.processTree(commit.tree,1) 105 | return clueCount -------------------------------------------------------------------------------- /Prospector.py: -------------------------------------------------------------------------------- 1 | # Prospector handles processing the job queue. This will use a synchronized queue so we can process jobs in parallel 2 | 3 | class Prospector: 4 | 5 | def __init__(self): 6 | self.jobQueue = Queue() 7 | 8 | def addJob(self, job): 9 | # Add a job object to the end of the queue 10 | self.jobQueue.put(job) 11 | 12 | def getJob(self): 13 | # Allow a worker to get the next available job. 14 | return self.jobQueue.get() 15 | 16 | def jobsDone(self): 17 | # indicate that a job is finished. We can remove it from the queue and mark it as finished in the crate. 18 | self.jobQueue.task_done() 19 | 20 | def loadJobs(self, crate): 21 | # Load jobs from a crate into the job queue 22 | return 23 | 24 | def postJobs(self, crate): 25 | # Save job queue to a crate, in the event of a shutdown or crash. 26 | return -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitLoot 2 | 3 | This tool began as a red team project to simply automate gitrob against our enterprise Github installation so that we could broadly scan to find potentially valuable information. After a bit of work, I had built an automated tool that collects a list of all users and organizations and passes them along to gitrob for the actual analysis. But when I finished this, I noticed several issues I was having with gitrob during the analysis stage, and the web app simply crumbled under the weight of the number of repositories we were assessing. Gitrob hasn't seen an update in six months, and I don't really enjoy writing ruby, so I decided I'd start my own implementation. 4 | 5 | ## Design goals: 6 | - Extensibility. 7 | - I want to be able to add new ways to store the data, and new ways to collect the data, with relative ease. 8 | - Historical analysis. 9 | - I want to be able to scan previous commit objects for potentially sensitive information leaks. 10 | - An option to "Quickscan" and only look at the repo HEAD, for faster scans. 11 | - Recurring analysis. 12 | - I want to regularly (and automatically) followup on repositories that I've been targeting. 13 | - An option to set the update interval 14 | - Local analysis. 15 | - I'd prefer to clone the repository locally for analysis to reduce the number of api requests I need to make. 16 | - An option to maintain local copies of repositories we add. 17 | - Whole API analysis 18 | - An easy way to automatically analyze the entire contents of an API. This is primarily useful for enterprise github installations. 19 | 20 | 21 | ## Clues 22 | 23 | A clue is how you define the types of information you want to look for. Clues get used by the metal detector to scan commits. A clue is a json file indicates the following: 24 | - Part of the file to look at 25 | - Extension 26 | - Filename 27 | - Content 28 | - Path 29 | - Commit Message 30 | - Type of matching to do 31 | - Match 32 | - regex 33 | - Pattern 34 | - Exact match on filename or extension 35 | - Regex match on extension, filename, content, commit message 36 | - Caption 37 | - Short description of what this clue looks for 38 | - Description 39 | - Why you are looking for this clue 40 | 41 | I'm using this design so that you can import existing signatures from [gitrob](https://github.com/michenriksen/gitrob/blob/master/signatures.json) 42 | 43 | 44 | ## Crates 45 | 46 | A crate is how we store our findings. It needs to be able to accept users, organizations, repositories, and loot. It implements the following: 47 | - addUser(self, user) 48 | - addOrganization(self, org) 49 | - addRepository(self, repo) 50 | - addLoot(self, loot) 51 | 52 | 53 | ## Loot 54 | 55 | Loot represents potentially valuable information, as defined by our clues. Loot is created by the metal detector and then passed to a crate. 56 | 57 | ## Maps 58 | 59 | A map is how we know how to communicate with the api. It needs to be able to fetch information about a user, an organization, and a repository based on a given string. A map also handles cycling out API keys when rate limits are hit. It implements the following: 60 | - getUser(self, username) 61 | - getOrganization(self, orgName) 62 | - getRepository(self, repoName) 63 | 64 | 65 | ## Metal Detector 66 | 67 | The metal detector inspects commit objects, looking for clues. If it finds clues, it creates loot and puts it in a crate. 68 | 69 | 70 | ## Models 71 | 72 | Models represent entities which we act upon to store and move data around the application. 73 | 74 | 75 | ## Prospector 76 | 77 | The prospector handles the job queue, dispatching jobs for processing as they are ready. 78 | 79 | 80 | ## Shovels 81 | 82 | A shovel is how we interact with a repository, iterating through commits and passing them to the metal detector. It takes a repository model at initialization and clones the repo locally. It creates commit objects to be passed to the metal detector. My initial plan is only to support git, however I'm designing it to (more) easily expand to other repo types if desirable. 83 | -------------------------------------------------------------------------------- /Shovel.py: -------------------------------------------------------------------------------- 1 | # Shovel abstraction allows us to interact with shovels without caring about the type of repository. 2 | import shovels 3 | 4 | class Shovel: 5 | def __init__(self, shovelType): 6 | if shovelType == "git": 7 | self._shovel = shovels.GitShovel() 8 | 9 | def setRepo(self, repo): 10 | self._shovel.setRepo(repo) 11 | 12 | def cleanUp(self): 13 | self._shovel.cleanUp() 14 | 15 | def clone(self): 16 | #clone the repo locally so that we can interact with it 17 | self._shovel.clone() 18 | 19 | def getHeadCommit(self): 20 | return self._shovel.getHeadCommit() 21 | 22 | def printRepo(self): 23 | self._shovel.printRepo() 24 | return 25 | 26 | def nextCommit(self): 27 | #get the next commit for inspection 28 | return self._shovel.nextCommit() -------------------------------------------------------------------------------- /clues/contents.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "part": "content", 4 | "type": "match", 5 | "pattern": "password=", 6 | "caption": "Potential password", 7 | "description": null 8 | } 9 | ] -------------------------------------------------------------------------------- /clues/extensions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "part": "extension", 4 | "type": "match", 5 | "pattern": "pem", 6 | "caption": "Potential cryptographic private key", 7 | "description": null 8 | }, 9 | { 10 | "part": "extension", 11 | "type": "regex", 12 | "pattern": "\\Akey(pair)?\\z", 13 | "caption": "Potential cryptographic private key", 14 | "description": null 15 | }, 16 | { 17 | "part": "extension", 18 | "type": "match", 19 | "pattern": "pkcs12", 20 | "caption": "Potential cryptographic key bundle", 21 | "description": null 22 | }, 23 | { 24 | "part": "extension", 25 | "type": "match", 26 | "pattern": "pfx", 27 | "caption": "Potential cryptographic key bundle", 28 | "description": null 29 | }, 30 | { 31 | "part": "extension", 32 | "type": "match", 33 | "pattern": "p12", 34 | "caption": "Potential cryptographic key bundle", 35 | "description": null 36 | }, 37 | { 38 | "part": "extension", 39 | "type": "match", 40 | "pattern": "asc", 41 | "caption": "Potential cryptographic key bundle", 42 | "description": null 43 | }, 44 | { 45 | "part": "extension", 46 | "type": "match", 47 | "pattern": "ovpn", 48 | "caption": "OpenVPN client configuration file", 49 | "description": null 50 | }, 51 | { 52 | "part": "extension", 53 | "type": "match", 54 | "pattern": "kdb", 55 | "caption": "KeePass password manager database file", 56 | "description": null 57 | }, 58 | { 59 | "part": "extension", 60 | "type": "match", 61 | "pattern": "agilekeychain", 62 | "caption": "1Password password manager database file", 63 | "description": null 64 | }, 65 | { 66 | "part": "extension", 67 | "type": "match", 68 | "pattern": "keychain", 69 | "caption": "Apple Keychain database file", 70 | "description": null 71 | }, 72 | { 73 | "part": "extension", 74 | "type": "regex", 75 | "pattern": "\\Akey(store|ring)\\z", 76 | "caption": "GNOME Keyring database file", 77 | "description": null 78 | }, 79 | { 80 | "part": "extension", 81 | "type": "match", 82 | "pattern": "log", 83 | "caption": "Log file", 84 | "description": "Log files might contain information such as references to secret HTTP endpoints, session IDs, user information, passwords and API keys." 85 | }, 86 | { 87 | "part": "extension", 88 | "type": "match", 89 | "pattern": "pcap", 90 | "caption": "Network traffic capture file", 91 | "description": null 92 | }, 93 | { 94 | "part": "extension", 95 | "type": "regex", 96 | "pattern": "\\Asql(dump)?\\z", 97 | "caption": "SQL dump file", 98 | "description": null 99 | }, 100 | { 101 | "part": "extension", 102 | "type": "match", 103 | "pattern": "gnucash", 104 | "caption": "GnuCash database file", 105 | "description": null 106 | }, 107 | { 108 | "part": "extension", 109 | "type": "match", 110 | "pattern": "kwallet", 111 | "caption": "KDE Wallet Manager database file", 112 | "description": null 113 | }, 114 | { 115 | "part": "extension", 116 | "type": "match", 117 | "pattern": "tblk", 118 | "caption": "Tunnelblick VPN configuration file", 119 | "description": null 120 | }, 121 | { 122 | "part": "extension", 123 | "type": "match", 124 | "pattern": "dayone", 125 | "caption": "Day One journal file", 126 | "description": null 127 | } 128 | ] -------------------------------------------------------------------------------- /clues/filenames.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "part": "filename", 4 | "type": "regex", 5 | "pattern": "\\A.*_rsa\\z", 6 | "caption": "Private SSH key", 7 | "description": null 8 | }, 9 | { 10 | "part": "filename", 11 | "type": "regex", 12 | "pattern": "\\A.*_dsa\\z", 13 | "caption": "Private SSH key", 14 | "description": null 15 | }, 16 | { 17 | "part": "filename", 18 | "type": "regex", 19 | "pattern": "\\A.*_ed25519\\z", 20 | "caption": "Private SSH key", 21 | "description": null 22 | }, 23 | { 24 | "part": "filename", 25 | "type": "regex", 26 | "pattern": "\\A.*_ecdsa\\z", 27 | "caption": "Private SSH key", 28 | "description": null 29 | }, 30 | { 31 | "part": "filename", 32 | "type": "match", 33 | "pattern": "otr.private_key", 34 | "caption": "Pidgin OTR private key", 35 | "description": null 36 | }, 37 | { 38 | "part": "filename", 39 | "type": "regex", 40 | "pattern": "\\A\\.?(bash_|zsh_|z)?history\\z", 41 | "caption": "Shell command history file", 42 | "description": null 43 | }, 44 | { 45 | "part": "filename", 46 | "type": "regex", 47 | "pattern": "\\A\\.?mysql_history\\z", 48 | "caption": "MySQL client command history file", 49 | "description": null 50 | }, 51 | { 52 | "part": "filename", 53 | "type": "regex", 54 | "pattern": "\\A\\.?psql_history\\z", 55 | "caption": "PostgreSQL client command history file", 56 | "description": null 57 | }, 58 | { 59 | "part": "filename", 60 | "type": "regex", 61 | "pattern": "\\A\\.?pgpass\\z", 62 | "caption": "PostgreSQL password file", 63 | "description": null 64 | }, 65 | { 66 | "part": "filename", 67 | "type": "regex", 68 | "pattern": "\\A\\.?irb_history\\z", 69 | "caption": "Ruby IRB console history file", 70 | "description": null 71 | }, 72 | { 73 | "part": "filename", 74 | "type": "regex", 75 | "pattern": "\\A\\.?dbeaver-data-sources.xml\\z", 76 | "caption": "DBeaver SQL database manager configuration file", 77 | "description": null 78 | }, 79 | { 80 | "part": "filename", 81 | "type": "regex", 82 | "pattern": "\\A\\.?muttrc\\z", 83 | "caption": "Mutt e-mail client configuration file", 84 | "description": null 85 | }, 86 | { 87 | "part": "filename", 88 | "type": "regex", 89 | "pattern": "\\A\\.?s3cfg\\z", 90 | "caption": "S3cmd configuration file", 91 | "description": null 92 | }, 93 | { 94 | "part": "filename", 95 | "type": "regex", 96 | "pattern": "\\A\\.?trc\\z", 97 | "caption": "T command-line Twitter client configuration file", 98 | "description": null 99 | }, 100 | { 101 | "part": "filename", 102 | "type": "regex", 103 | "pattern": "\\A\\.?gitrobrc\\z", 104 | "caption": "Well, this is awkward... Gitrob configuration file", 105 | "description": null 106 | }, 107 | { 108 | "part": "filename", 109 | "type": "regex", 110 | "pattern": "\\A\\.?(bash|zsh)rc\\z", 111 | "caption": "Shell configuration file", 112 | "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys." 113 | }, 114 | { 115 | "part": "filename", 116 | "type": "regex", 117 | "pattern": "\\A\\.?(bash_|zsh_)?profile\\z", 118 | "caption": "Shell profile configuration file", 119 | "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys." 120 | }, 121 | { 122 | "part": "filename", 123 | "type": "regex", 124 | "pattern": "\\A\\.?(bash_|zsh_)?aliases\\z", 125 | "caption": "Shell command alias configuration file", 126 | "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys." 127 | }, 128 | { 129 | "part": "filename", 130 | "type": "match", 131 | "pattern": "secret_token.rb", 132 | "caption": "Ruby On Rails secret token configuration file", 133 | "description": "If the Rails secret token is known, it can allow for remote code execution. (http://www.exploit-db.com/exploits/27527/)" 134 | }, 135 | { 136 | "part": "filename", 137 | "type": "match", 138 | "pattern": "omniauth.rb", 139 | "caption": "OmniAuth configuration file", 140 | "description": "The OmniAuth configuration file might contain client application secrets." 141 | }, 142 | { 143 | "part": "filename", 144 | "type": "match", 145 | "pattern": "carrierwave.rb", 146 | "caption": "Carrierwave configuration file", 147 | "description": "Can contain credentials for online storage systems such as Amazon S3 and Google Storage." 148 | }, 149 | { 150 | "part": "filename", 151 | "type": "match", 152 | "pattern": "schema.rb", 153 | "caption": "Ruby On Rails database schema file", 154 | "description": "Contains information on the database schema of a Ruby On Rails application." 155 | }, 156 | { 157 | "part": "filename", 158 | "type": "match", 159 | "pattern": "database.yml", 160 | "caption": "Potential Ruby On Rails database configuration file", 161 | "description": "Might contain database credentials." 162 | }, 163 | { 164 | "part": "filename", 165 | "type": "match", 166 | "pattern": "settings.py", 167 | "caption": "Django configuration file", 168 | "description": "Might contain database credentials, online storage system credentials, secret keys, etc." 169 | }, 170 | { 171 | "part": "filename", 172 | "type": "regex", 173 | "pattern": "\\A(.*)?config(\\.inc)?\\.php\\z", 174 | "caption": "PHP configuration file", 175 | "description": "Might contain credentials and keys." 176 | }, 177 | { 178 | "part": "filename", 179 | "type": "regex", 180 | "pattern": "backup", 181 | "caption": "Contains word: backup", 182 | "description": null 183 | }, 184 | { 185 | "part": "filename", 186 | "type": "regex", 187 | "pattern": "dump", 188 | "caption": "Contains word: dump", 189 | "description": null 190 | }, 191 | { 192 | "part": "filename", 193 | "type": "regex", 194 | "pattern": "password", 195 | "caption": "Contains word: password", 196 | "description": null 197 | }, 198 | { 199 | "part": "filename", 200 | "type": "regex", 201 | "pattern": "credential", 202 | "caption": "Contains word: credential", 203 | "description": null 204 | }, 205 | { 206 | "part": "filename", 207 | "type": "regex", 208 | "pattern": "secret", 209 | "caption": "Contains word: secret", 210 | "description": null 211 | }, 212 | { 213 | "part": "filename", 214 | "type": "regex", 215 | "pattern": "private.*key", 216 | "caption": "Contains words: private, key", 217 | "description": null 218 | }, 219 | { 220 | "part": "filename", 221 | "type": "match", 222 | "pattern": "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", 223 | "caption": "Jenkins publish over SSH plugin file", 224 | "description": null 225 | }, 226 | { 227 | "part": "filename", 228 | "type": "match", 229 | "pattern": "credentials.xml", 230 | "caption": "Potential Jenkins credentials file", 231 | "description": null 232 | }, 233 | { 234 | "part": "filename", 235 | "type": "regex", 236 | "pattern": "\\A\\.?htpasswd\\z", 237 | "caption": "Apache htpasswd file", 238 | "description": null 239 | }, 240 | { 241 | "part": "filename", 242 | "type": "regex", 243 | "pattern": "\\A(\\.|_)?netrc\\z", 244 | "caption": "Configuration file for auto-login process", 245 | "description": "Might contain username and password." 246 | }, 247 | { 248 | "part": "filename", 249 | "type": "match", 250 | "pattern": "LocalSettings.php", 251 | "caption": "Potential MediaWiki configuration file", 252 | "description": null 253 | }, 254 | { 255 | "part": "filename", 256 | "type": "regex", 257 | "pattern": "\\A\\.pubxml(\\.user)?\\z", 258 | "caption": "Potential MSBuild publish profile", 259 | "description": null 260 | }, 261 | { 262 | "part": "filename", 263 | "type": "match", 264 | "pattern": "Favorites.plist", 265 | "caption": "Sequel Pro MySQL database manager bookmark file", 266 | "description": null 267 | }, 268 | { 269 | "part": "filename", 270 | "type": "match", 271 | "pattern": "configuration.user.xpl", 272 | "caption": "Little Snitch firewall configuration file", 273 | "description": "Contains traffic rules for applications" 274 | }, 275 | { 276 | "part": "filename", 277 | "type": "match", 278 | "pattern": "journal.txt", 279 | "caption": "Potential jrnl journal file", 280 | "description": null 281 | }, 282 | { 283 | "part": "filename", 284 | "type": "regex", 285 | "pattern": "\\A\\.?tugboat\\z", 286 | "caption": "Tugboat DigitalOcean management tool configuration", 287 | "description": null 288 | }, 289 | { 290 | "part": "filename", 291 | "type": "regex", 292 | "pattern": "\\A\\.?git-credentials\\z", 293 | "caption": "git-credential-store helper credentials file", 294 | "description": null 295 | }, 296 | { 297 | "part": "filename", 298 | "type": "regex", 299 | "pattern": "\\A\\.?gitconfig\\z", 300 | "caption": "Git configuration file", 301 | "description": null 302 | }, 303 | { 304 | "part": "filename", 305 | "type": "match", 306 | "pattern": "knife.rb", 307 | "caption": "Chef Knife configuration file", 308 | "description": "Might contain references to Chef servers" 309 | }, 310 | { 311 | "part": "filename", 312 | "type": "match", 313 | "pattern": "proftpdpasswd", 314 | "caption": "cPanel backup ProFTPd credentials file", 315 | "description": "Contains usernames and password hashes for FTP accounts" 316 | }, 317 | { 318 | "part": "filename", 319 | "type": "match", 320 | "pattern": "robomongo.json", 321 | "caption": "Robomongo MongoDB manager configuration file", 322 | "description": "Might contain credentials for MongoDB databases" 323 | }, 324 | { 325 | "part": "filename", 326 | "type": "match", 327 | "pattern": "filezilla.xml", 328 | "caption": "FileZilla FTP configuration file", 329 | "description": "Might contain credentials for FTP servers" 330 | }, 331 | { 332 | "part": "filename", 333 | "type": "match", 334 | "pattern": "recentservers.xml", 335 | "caption": "FileZilla FTP recent servers file", 336 | "description": "Might contain credentials for FTP servers" 337 | }, 338 | { 339 | "part": "filename", 340 | "type": "match", 341 | "pattern": "ventrilo_srv.ini", 342 | "caption": "Ventrilo server configuration file", 343 | "description": "Might contain passwords" 344 | }, 345 | { 346 | "part": "filename", 347 | "type": "regex", 348 | "pattern": "\\A\\.?dockercfg\\z", 349 | "caption": "Docker configuration file", 350 | "description": "Might contain credentials for public or private Docker registries" 351 | }, 352 | { 353 | "part": "filename", 354 | "type": "regex", 355 | "pattern": "\\A\\.?npmrc\\z", 356 | "caption": "NPM configuration file", 357 | "description": "Might contain credentials for NPM registries" 358 | }, 359 | { 360 | "part": "filename", 361 | "type": "match", 362 | "pattern": "terraform.tfvars", 363 | "caption": "Terraform variable config file", 364 | "description": "Might contain credentials for terraform providers" 365 | }, 366 | { 367 | "part": "filename", 368 | "type": "regex", 369 | "pattern": "\\A\\.?env\\z", 370 | "caption": "Environment configuration file", 371 | "description": null 372 | } 373 | 374 | ] -------------------------------------------------------------------------------- /clues/messages.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "part": "message", 4 | "type": "match", 5 | "pattern": "removed password", 6 | "caption": "We might find a password near this commit", 7 | "description": null 8 | }, 9 | { 10 | "part": "message", 11 | "type": "match", 12 | "pattern": "Ubuntu", 13 | "caption": "I don't actually care about this", 14 | "description": null 15 | } 16 | ] -------------------------------------------------------------------------------- /clues/paths.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "part": "path", 4 | "type": "regex", 5 | "pattern": "\\.?ssh/config\\z", 6 | "caption": "SSH configuration file", 7 | "description": null 8 | }, 9 | { 10 | "part": "path", 11 | "type": "regex", 12 | "pattern": "\\.?purple\\/accounts\\.xml\\z", 13 | "caption": "Pidgin chat client account configuration file", 14 | "description": null 15 | }, 16 | { 17 | "part": "path", 18 | "type": "regex", 19 | "pattern": "\\.?xchat2?\\/servlist_?\\.conf\\z", 20 | "caption": "Hexchat/XChat IRC client server list configuration file", 21 | "description": null 22 | }, 23 | { 24 | "part": "path", 25 | "type": "regex", 26 | "pattern": "\\.?irssi\\/config\\z", 27 | "caption": "Irssi IRC client configuration file", 28 | "description": null 29 | }, 30 | { 31 | "part": "path", 32 | "type": "regex", 33 | "pattern": "\\.?recon-ng\\/keys\\.db\\z", 34 | "caption": "Recon-ng web reconnaissance framework API key database", 35 | "description": null 36 | }, 37 | 38 | { 39 | "part": "path", 40 | "type": "regex", 41 | "pattern": "\\.?aws/credentials\\z", 42 | "caption": "AWS CLI credentials file", 43 | "description": null 44 | }, 45 | { 46 | "part": "path", 47 | "type": "regex", 48 | "pattern": "\\.?gem/credentials\\z", 49 | "caption": "Rubygems credentials file", 50 | "description": "Might contain API key for a rubygems.org account." 51 | }, 52 | { 53 | "part": "path", 54 | "type": "regex", 55 | "pattern": "\\.?chef/(.*)\\.pem\\z", 56 | "caption": "Chef private key", 57 | "description": "Can be used to authenticate against Chef servers" 58 | } 59 | ] -------------------------------------------------------------------------------- /crates/FileCrate.py: -------------------------------------------------------------------------------- 1 | # Used to store loot in files. 2 | 3 | class FileCrate: 4 | def __init__(self): 5 | self.rootDir = "." 6 | 7 | 8 | def addUser(self, user): 9 | #add a user file to our directory. domain/user 10 | return 11 | 12 | def addOrganization(self, org): 13 | #add an org file to our directory. domain/org 14 | #keep a list of users inside our org file. 15 | return 16 | 17 | def addRepository(self, repo): 18 | #add a repo to our directory domain/(user|org)/reponame 19 | return 20 | 21 | def addLoot(self, loot): 22 | #add loot to the loot directory. Store under domain/loot/reponame/timestamp-loottype.json 23 | return 24 | 25 | def addJob(self,job): 26 | #add job to a jobs directory. /domain/jobs/jobnum 27 | return 28 | 29 | def finishJob(self,job): 30 | # This job is finished, let's update the record and set a finish time, create a new job w/ start time = finishTime + rescanInterval 31 | return -------------------------------------------------------------------------------- /crates/SQLCrate.py: -------------------------------------------------------------------------------- 1 | # Connector for storing loot in MySQL databases. 2 | from ConfigParser import ConfigParser 3 | class SQLCrate: 4 | def __init__(self, config=None): 5 | if config: 6 | self.CONFIG_FILE = config 7 | else: 8 | self.CONFIG_FILE = None 9 | self.db_name = "" 10 | self.db_user = "" 11 | self.db_pass = "" 12 | self.db_host = "" 13 | self.loadSettings() 14 | 15 | def loadSettings(self): 16 | # fetch settings from config file 17 | config = ConfigParser() 18 | if self.CONFIG_FILE: 19 | config.read(self.CONFIG_FILE) 20 | else: 21 | if not config.read(os.path.expanduser('~') + "/.gitloot"): 22 | raise IOError, "cannot load ~/.gitloot" 23 | confDict = dict(config.items(self.__class__.__name__)) 24 | if (confDict['db_name']): 25 | self.db_name = confDict['db_name'] 26 | if (confDict['db_user']): 27 | self.db_user = confDict['db_user'] 28 | if (confDict['db_pass']): 29 | self.db_pass = confDict['db_pass'] 30 | if (confDict['db_host']): 31 | self.db_host = confDict['db_host'] 32 | return 0 33 | 34 | def addUser(self, user): 35 | #add a user to the user table 36 | return 0 37 | 38 | def addOrganization(self, org): 39 | #add an org to the org table 40 | #add org<->member to a table that tracks members of an org 41 | return 0 42 | 43 | def addRepository(self, repo): 44 | #add a repo to repo table 45 | #foreign key on repoOwner to an org or user id 46 | return 0 47 | 48 | def addLoot(self, loot): 49 | #add loot to the loot table 50 | return 0 51 | 52 | def addJob(self,job): 53 | #add job to a jobs table. 54 | return 55 | 56 | def finishJob(self,job): 57 | # This job is finished, let's update the record and set a finish time, create a new job w/ start time = finishTime + rescanInterval 58 | return -------------------------------------------------------------------------------- /maps/BitbucketMap.py: -------------------------------------------------------------------------------- 1 | # Bitbucket api calls -------------------------------------------------------------------------------- /maps/GithubMap.py: -------------------------------------------------------------------------------- 1 | # Github api calls for common tasks 2 | import requests 3 | import os 4 | from ConfigParser import ConfigParser 5 | from models import Organization, Repository, User 6 | class GithubMap(object): 7 | 8 | def __init__(self, config=None): 9 | if config: 10 | self.CONFIG_FILE = config 11 | else: 12 | self.CONFIG_FILE = None 13 | self.next = "" 14 | self.API_ROOT = r"https://api.github.com/" # default api_root 15 | self.GET_USER = self.API_ROOT + "users/" 16 | self.GET_USERS = self.API_ROOT + "users" 17 | self.GET_ORG = self.API_ROOT + "orgs/" 18 | self.GET_REPO = self.API_ROOT + "repos/" 19 | self.TOKENS = [] 20 | self.current_token = 0 21 | self.user_agent = "0xdade/GitLoot" # Default user_agent 22 | self.loadSettings() 23 | self.HEADERS = {'user-agent': self.user_agent, 'Authorization': "token %s" % self.TOKENS[self.current_token]} 24 | return 25 | 26 | def loadSettings(self): 27 | # Load settings from config file 28 | config = ConfigParser() 29 | if self.CONFIG_FILE: 30 | config.read(self.CONFIG_FILE) 31 | else: 32 | if not config.read(os.path.expanduser('~') + "/.gitloot"): 33 | raise IOError, "cannot load ~/.gitloot" 34 | confDict = dict(config.items(self.__class__.__name__)) 35 | if (confDict['api_root']): 36 | self.API_ROOT = confDict['api_root'] 37 | if (confDict['user-agent']): 38 | self.user_agent = confDict['user-agent'] 39 | if (confDict['api_tokens']): 40 | self.TOKENS = confDict['api_tokens'].strip("\"").split(",") 41 | return 42 | 43 | def getUsers(self): 44 | if self.next != "": 45 | r = self.makeRequest(self.next) 46 | else: 47 | r = self.makeRequest(self.GET_USERS) 48 | self.next = r.links["next"]['url'] 49 | return r.json() 50 | 51 | def switchTokens(self): 52 | # switch to a new token 53 | if self.current_token == len(self.TOKENS)-1: 54 | self.current_token = 0 55 | elif self.current_token < len(self.TOKENS): 56 | self.current_token += 1 57 | self.HEADERS['Authorization'] = "token %s" % self.TOKENS[self.current_token] 58 | return 59 | 60 | def makeRequest(self, entity): 61 | r = requests.get(entity, headers=self.HEADERS) 62 | if (int(r.headers['X-RateLimit-Remaining']) < 10): 63 | self.switchTokens() 64 | if (r.status_code == 200): 65 | return r 66 | else: 67 | raise SystemExit(str(r.status_code) + ": Seems we've lost the way, sir.") 68 | 69 | def getUser(self, uname): 70 | # API call to fetch user 71 | r = self.makeRequest(self.GET_USER + uname) 72 | user = User(r.json()) 73 | user.repos = self.getUserRepos(user) 74 | return user 75 | 76 | def getUserRepos(self, user): 77 | # Pending multipage handling 78 | repos = {} 79 | r = self.makeRequest(user.reposUrl) 80 | for repo in r.json(): 81 | repos[repo['id']] = repo['full_name'] 82 | return repos 83 | 84 | def getOrgRepos(self, org): 85 | # Pending multipage handling 86 | repos = {} 87 | r = self.makeRequest(org.reposUrl) 88 | for repo in r.json(): 89 | repos[repo['id']] = repo['full_name'] 90 | return repos 91 | 92 | def getOrganization(self, orgName): 93 | # API call to fetch organization 94 | r = self.makeRequest(self.GET_ORG + orgName) 95 | org = Organization(r.json()) 96 | org.members = self.getOrganizationMembers(orgName) 97 | org.repos = self.getOrgRepos(org) 98 | return org 99 | 100 | def getOrganizationMembers(self,orgName): 101 | # API call to fetch list of public members of organization 102 | # Pending multipage handling 103 | members = {} 104 | r = self.makeRequest(self.GET_ORG + orgName + "/public_members") 105 | for member in r.json(): 106 | members[member['id']] = member['login'] 107 | return members 108 | 109 | def getRepository(self, full_name): 110 | # API call to fetch repository 111 | r = self.makeRequest(self.GET_REPO + full_name) 112 | repo = Repository(r.json()) 113 | return repo -------------------------------------------------------------------------------- /maps/__init__.py: -------------------------------------------------------------------------------- 1 | from GithubMap import * 2 | -------------------------------------------------------------------------------- /models/Commit.py: -------------------------------------------------------------------------------- 1 | class Commit: 2 | 3 | 4 | def __init__(self): 5 | # Init user object. 6 | self.sha = "" 7 | self.author = 0 8 | self.committer = 0 9 | self.repository = 0 10 | self.treeUrl = "" 11 | self.message = "" 12 | self.commitDate = "" -------------------------------------------------------------------------------- /models/Job.py: -------------------------------------------------------------------------------- 1 | # Jobs encapsulate the necessary information for the prospector to 2 | 3 | class Job: 4 | def __init__(self): 5 | self.createTime = 0 6 | self.finishTime = 0 7 | self.apiLink = "" # The link to the api root that we build our queries off of. 8 | self.apiKeys = [] # The (list of) API key(s) to use when processing the job 9 | self.subject = "" # The subject of the job. Could be a user, org, repo, or API. 10 | self.rescanInterval = 0 # an integer representing the number of hours to go between rescans. 11 | self.crateType = "" # this allows us to store different jobs with different methods. Useful for running a job if you only want data dumped to a file. 12 | self.jobType = "" # (User, Org, Repo, API) 13 | self.parent = 0 # Keep track of my parent job. If no parent, we can mark the job finished. 14 | -------------------------------------------------------------------------------- /models/Loot.py: -------------------------------------------------------------------------------- 1 | class Loot: 2 | 3 | 4 | def __init__(self): 5 | # Init user object. 6 | self.sourceRepo = 0 # remote ID number of repo 7 | self.sourceUrl = "" # remote url to repo 8 | self.sourceOwner = 0 # remote ID of user 9 | self.sourceHash = "" # Store the hash of the commit object we found this in 10 | self.author = "" # Author of the commit object we found loot in 11 | self.discoveryDate = 0 # Timestamp that we discovered this loot 12 | self.lootType = "" # Caption of the clue that characterized this as loot 13 | self.lootContents = "" # string representation of the loot found. -------------------------------------------------------------------------------- /models/Organization.py: -------------------------------------------------------------------------------- 1 | class Organization: 2 | 3 | def __init__(self,json): 4 | # Init user object. 5 | self.id = json['id'] 6 | self.login = json['login'] 7 | self.email = json['email'] 8 | self.blog = json['blog'] 9 | self.repoCount = json['public_repos'] 10 | self.url = json['url'] 11 | self.reposUrl = json['repos_url'] 12 | self.members = {} 13 | self.repos = {} 14 | 15 | def __str__(self): 16 | memberList = "" 17 | repoList = "" 18 | for member in self.members: 19 | memberList += str(member) + "," 20 | for repo in self.repos: 21 | repoList += str(repo) + "," 22 | return "Organization: " + str(self.id) + ", " + str(self.login) + "\nMembers: " + memberList[:-1] + "\nRepos: " + repoList[:-1] 23 | 24 | def getMembers(self): 25 | return self.members.items() 26 | 27 | def getRepos(self): 28 | return self.repos.items() -------------------------------------------------------------------------------- /models/Repository.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | class Repository: 3 | 4 | def __init__(self, json): 5 | self.id = json['id'] 6 | self.owner = json['owner']['id'] 7 | self.name = json['name'] 8 | self.full_name = json['full_name'] 9 | self.url = json['url'] 10 | self.cloneUrl = json['clone_url'] 11 | self.commits = deque() 12 | self.headCommit = "" 13 | 14 | def __str__(self): 15 | return "Repository: " + str(self.full_name) -------------------------------------------------------------------------------- /models/User.py: -------------------------------------------------------------------------------- 1 | class User: 2 | 3 | def __init__(self, json): 4 | # Init user object. 5 | self.id = json['id'] 6 | self.login = json['login'] 7 | self.email = json['email'] 8 | self.company = json['company'] 9 | self.blog = json['blog'] 10 | self.repoCount = json['public_repos'] 11 | self.url = json['url'] 12 | self.reposUrl = json['repos_url'] 13 | self.repos = {} 14 | 15 | def __str__(self): 16 | repoList = "" 17 | for repo in self.repos: 18 | repoList += str(repo) + "," 19 | return "User: " + str(self.id) + ", " + str(self.login) + ", " + str(self.repoCount) + "\nRepos: " + repoList[:-1] 20 | 21 | def getRepos(self): 22 | return self.repos.items() -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | from Commit import * 2 | from Job import * 3 | from Loot import * 4 | from Organization import * 5 | from Repository import * 6 | from User import * -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | GitPython==2.0.5 2 | requests==2.20.0 3 | argparse==1.2.1 4 | configparser==3.3.0.post2 5 | GitPython==2.0.5 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Check if prereqs are installed 2 | # Set persistent information in config 3 | from ConfigParser import ConfigParser 4 | import os 5 | import readline 6 | class Setup: 7 | 8 | def __init__(self): 9 | # Do all the setup on setup instantiation 10 | self.setupDependencies() 11 | self.setupMap() 12 | self.setupCrate() 13 | return 14 | 15 | def printHeader(self, text): 16 | print ("#"*10) + "\n" + text + "\n" + ("#"* 10) 17 | 18 | def setupCrate(self): 19 | # initialize the crate matching crateType 20 | config = ConfigParser() 21 | config.read(os.path.expanduser('~') + "/.gitloot") 22 | try: 23 | crateDict = dict(config.items("SQLCrate")) 24 | return 25 | except: 26 | self.printHeader("CRATE SETUP") 27 | crate_type = "sql" 28 | db_name = raw_input("Database name: ") 29 | db_user = raw_input("Database user: ") 30 | db_pass = raw_input("Database pass: ") 31 | db_host = raw_input("Database host: ") 32 | 33 | configFile = os.path.expanduser('~') + "/.gitloot" 34 | if crate_type.lower() == "sql": 35 | confHeader = "\n[SQLCrate]\n" 36 | else: 37 | print "We only support SQL crates right now." 38 | confHeader = "\n[SQLCrate]\n" 39 | with open(configFile, 'a') as conf: 40 | conf.write(confHeader + "db_name=%s\ndb_user=%s\ndb_pass=%s\ndb_host=%s\n" % (db_name, db_user, db_pass, db_host)) 41 | return 42 | 43 | 44 | 45 | def setupMap(self): 46 | # Ask the user for details about how they are going to run the tool 47 | # Save this to ~/.gitloot 48 | config = ConfigParser() 49 | config.read(os.path.expanduser('~') + "/.gitloot") 50 | try: 51 | mapDict = dict(config.items("GithubMap")) 52 | return 53 | except: 54 | self.printHeader("MAP SETUP") 55 | #print "What type of API is this? (github)" 56 | #api_type = raw_input("API Type: ") 57 | api_type = "github" # Commented out the question about this, it's unnecessary at the moment. 58 | print "Please provide the API root URI that we'll be running against. (https://api.github.com)" 59 | api_root = raw_input("API Root: ") 60 | if api_root[-1] != "/": 61 | api_root = api_root + "/" 62 | print "Please enter API tokens as a comma separated list" 63 | api_tokens = raw_input("API Token(s): ") 64 | api_tokens = api_tokens.strip(" ") 65 | print "What user agent would you like to use?" 66 | user_agent = raw_input("User-Agent: ") 67 | configFile = os.path.expanduser('~') + "/.gitloot" 68 | 69 | if api_type.lower() == "github": 70 | confHeader = "[GithubMap]\n" 71 | else: 72 | print "We only support github APIs right now." 73 | confHeader = "\n[GithubMap]\n" 74 | with open(configFile, 'a') as conf: 75 | conf.write(confHeader + "api_root=\"%s\"\napi_tokens=\"%s\"\nuser-agent=\"%s\"\n" % (api_root, api_tokens, user_agent)) 76 | return 77 | 78 | def setupDependencies(self): 79 | # pip install the requirements.txt file 80 | try: 81 | import pip 82 | self.printHeader("DEPENDENCIES") 83 | pip.main(['install', '-r', 'requirements.txt']) 84 | except ImportError: 85 | print "Please install pip to continue. . ." 86 | raise SystemExit(1) 87 | return 88 | 89 | def main(): 90 | # setup.py can run standalone or run automatically at first run. 91 | setup = Setup() 92 | 93 | if __name__ == '__main__': 94 | main() -------------------------------------------------------------------------------- /shovels/GitShovel.py: -------------------------------------------------------------------------------- 1 | # Git Shovel handles interacting with git repositories. Probably going to use GitPython 2 | # The shovel combs through a repository and creates commit objects to be analyzed 3 | import git 4 | import shutil 5 | class GitShovel: 6 | def __init__(self): 7 | self.path = "workbench/" 8 | self.repo = "" 9 | self.cloned_repo = None 10 | self.commits = None 11 | return 12 | 13 | def setRepo(self, repo): 14 | self.repo = repo 15 | self.path = "workbench/" + repo.full_name 16 | 17 | def clone(self): 18 | #clone the repo locally so that we can interact with it 19 | self.cloned_repo = git.Repo.clone_from(self.repo.cloneUrl, self.path)#, depth=1) 20 | self.commits = self.cloned_repo.iter_commits() 21 | return 22 | 23 | def cleanUp(self): 24 | self.repo = None 25 | shutil.rmtree(self.path) 26 | self.path = "workbench/" 27 | self.cloned_repo = None 28 | 29 | def getHeadCommit(self): 30 | return self.cloned_repo.head.commit 31 | 32 | def printRepo(self): 33 | for commit in self.commits: 34 | print commit 35 | 36 | def nextCommit(self): 37 | #get the next commit for inspection 38 | for commit in self.commits: 39 | yield commit 40 | raise StopIteration -------------------------------------------------------------------------------- /shovels/__init__.py: -------------------------------------------------------------------------------- 1 | from GitShovel import * --------------------------------------------------------------------------------