├── .gitignore ├── Makefile ├── assets ├── 0.5x │ └── gorgit-logo@0.5x.png └── 1x │ └── gorgit-logo.png ├── bin ├── go-rip-git └── go-rip-git.exe ├── go.mod ├── go.sum ├── main.go ├── readme.md └── scraper └── scraper.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build-release: 2 | rm -rf bin/* 3 | GOOS=linux GOARCH=386 go build -o bin/go-rip-git 4 | GOOS=windows GOARCH=386 go build -o bin/go-rip-git.exe 5 | -------------------------------------------------------------------------------- /assets/0.5x/gorgit-logo@0.5x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RonniSkansing/go-rip-git/b92b72c30c494db362fcc348e19907dde27370fa/assets/0.5x/gorgit-logo@0.5x.png -------------------------------------------------------------------------------- /assets/1x/gorgit-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RonniSkansing/go-rip-git/b92b72c30c494db362fcc348e19907dde27370fa/assets/1x/gorgit-logo.png -------------------------------------------------------------------------------- /bin/go-rip-git: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RonniSkansing/go-rip-git/b92b72c30c494db362fcc348e19907dde27370fa/bin/go-rip-git -------------------------------------------------------------------------------- /bin/go-rip-git.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RonniSkansing/go-rip-git/b92b72c30c494db362fcc348e19907dde27370fa/bin/go-rip-git.exe -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/RonniSkansing/go-rip-git 2 | 3 | require golang.org/x/net v0.0.0-20190110200230-915654e7eabc 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM= 2 | golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 3 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/RonniSkansing/go-rip-git/scraper" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | ) 11 | 12 | func main() { 13 | var ( 14 | target = flag.String("u", "", "URL to scan") 15 | scrape = flag.Bool("s", false, "scrape source files") 16 | idleConnTimeout = flag.Int("t", 5, "request connection idle timeout in seconds") 17 | gitPath = flag.String("p", "/.git/", "the absolute path to the git folder") 18 | concurrency = flag.Int("c", 100, "concurrent scrape requests") 19 | wait = flag.Duration("w", 0 * time.Second, "time in seconds to wait between each request, example 5s") 20 | veryVerbose = flag.Bool("vv", false, "very verbose output") 21 | ) 22 | flag.Parse() 23 | 24 | if len(*target) == 0 { 25 | flag.PrintDefaults() 26 | return 27 | } 28 | 29 | c := scraper.Config{ 30 | ConcurrentRequests: *concurrency, 31 | WaitTimeBetweenRequest: *wait, 32 | VeryVerbose: *veryVerbose, 33 | } 34 | sr := scraper.NewScraper( 35 | &http.Client{Timeout: time.Duration(*idleConnTimeout) * time.Second}, 36 | &c, 37 | func(err error) { 38 | log.Printf("scrape error: %v", err) 39 | }, 40 | ) 41 | uri, err := url.ParseRequestURI(*target + *gitPath) 42 | if err != nil { 43 | log.Fatalf("invalid URL: %v", err) 44 | } 45 | if *scrape { 46 | err := sr.Scrape(uri) 47 | if err != nil { 48 | log.Fatalf("failed to scrape: %v", err) 49 | } 50 | } else { 51 | entries, err := sr.GetEntries(uri) 52 | if err != nil { 53 | log.Fatalf("failed to get index entries: %v", err) 54 | } 55 | log.Println("Contents of " + uri.String()) 56 | for i := 0; i < len(entries); i++ { 57 | log.Println(entries[i].Sha + " " + entries[i].FileName) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ![Gopher robbing git](https://raw.githubusercontent.com/RonnieSkansing/gorgit/master/assets/0.5x/gorgit-logo%400.5x.png) 2 | 3 | git file lister and source scraper 4 | 5 | Not for use on git servers but for deploys accidentally including the `.git` folder 6 | 7 | Zero dependencies and does not require the target to have open directory listing. 8 | 9 | # Usage 10 | 11 | ``` 12 | Usage of go-rip-git: 13 | -c int 14 | concurrent scrape requests (default 100) 15 | -p string 16 | the absolute path to the git folder (default "/.git/") 17 | -s scrape source files 18 | -t int 19 | request connection idle timeout in seconds (default 5) 20 | -u string 21 | URL to scan 22 | -vv 23 | very verbose output 24 | -w duration 25 | time in seconds to wait between each request, example 5s 26 | ``` 27 | 28 | ### Show files 29 | `go-rip-git -u http://target.tld` 30 | 31 | Results in something like 32 | ``` 33 | c1f3161c27b7fb86615a4916f595473a0a76c774 .env 34 | 29c16c3f37ea57569fbf9cc1ce183938a9710aed config/config.json 35 | ... 36 | ``` 37 | 38 | ## Proxy 39 | HTTP_PROXY="socks5://127.0.0.1:9150/" go-rip-git -u http://target.tld 40 | 41 | HTTPS_PROXY="socks5://127.0.0.1:9150/" go-rip-git -u https://target.tld 42 | 43 | ## Scrape files 44 | `go-rip-git -u http://target.tld -s` 45 | 46 | Scraped source is found in `target.tld/...`` 47 | 48 | # Developer notes / TODO 49 | Pull requests with features, fixes and refactoring are appreciated 50 | 51 | Things that come into mind 52 | - Extract contents of .PACK files 53 | - Choose output directory 54 | - Tests 55 | - Accepting a list of targets (from arg and file) 56 | 57 | Found a **bug**? Create an issue 58 | 59 | # Credits 60 | ### Logo 61 | Thanks to [Paula Sobczak](https://paulajs.dk) for logo based on [Renee French's gophers](http://reneefrench.blogspot.com/) 62 | licensed licensedhttps://creativecommons.org/licenses/by/3.0/ 63 | 64 | # Disclaimer 65 | *Author accepts no liability and no responsibility for the use of this tool. It is intended only for use with consent.* 66 | -------------------------------------------------------------------------------- /scraper/scraper.go: -------------------------------------------------------------------------------- 1 | package scraper 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "encoding/hex" 8 | "fmt" 9 | "io/ioutil" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "path" 14 | "path/filepath" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | type ErrorHandler = func(error) 20 | 21 | // Scraper Scrapes git 22 | type Scraper struct { 23 | client *http.Client 24 | config *Config 25 | errorHandler ErrorHandler 26 | } 27 | 28 | type Config struct { 29 | ConcurrentRequests int 30 | WaitTimeBetweenRequest time.Duration 31 | VeryVerbose bool 32 | } 33 | 34 | // IdxEntry is a map between the Sha and the file it points to 35 | type IdxEntry struct { 36 | Sha string 37 | FileName string 38 | } 39 | 40 | // NewScraper Creates a new scraper instance pointer 41 | func NewScraper(client *http.Client, config *Config, errHandler ErrorHandler) *Scraper { 42 | return &Scraper{ 43 | client: client, 44 | config: config, 45 | errorHandler: errHandler, 46 | } 47 | } 48 | 49 | // getIndexFile retrieves the git index file as a byte slice 50 | func (s *Scraper) getIndexFile(target *url.URL) ([]byte, error) { 51 | res, err := s.client.Get(target.String() + "index") 52 | if err != nil { 53 | return nil, err 54 | } 55 | defer res.Body.Close() 56 | if res.StatusCode != http.StatusOK { 57 | return nil, fmt.Errorf("failed to get index file: %s", res.Status) 58 | } 59 | 60 | return ioutil.ReadAll(res.Body) 61 | } 62 | 63 | // Scrape parses remote git index and converts each listed file to source locally 64 | func (s *Scraper) Scrape(target *url.URL) error { 65 | h := target.Hostname() 66 | if err := os.MkdirAll(h, os.ModePerm); err != nil { 67 | return fmt.Errorf("failed to create scrape result folder: %v", err) 68 | } 69 | entries, err := s.GetEntries(target) 70 | time.Sleep(s.config.WaitTimeBetweenRequest) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | throttle := sync.WaitGroup{} 76 | untilDone := sync.WaitGroup{} 77 | onDone := func() { 78 | untilDone.Done() 79 | } 80 | for i, j := 0, 1; i < len(entries); i,j = i+1, j+1{ 81 | untilDone.Add(1) 82 | if j >= s.config.ConcurrentRequests { 83 | throttle.Add(1) 84 | onDone = func() { 85 | throttle.Done() 86 | untilDone.Done() 87 | } 88 | } 89 | entry := entries[i] 90 | f := target.String() + "objects/" + entry.Sha[0:2] + "/" + entry.Sha[2:] 91 | go s.getAndPersist(f, filepath.Join(target.Hostname(), entry.FileName), onDone) 92 | time.Sleep(s.config.WaitTimeBetweenRequest) 93 | if j >= s.config.ConcurrentRequests { 94 | throttle.Wait() 95 | } 96 | } 97 | untilDone.Wait() 98 | 99 | return nil 100 | } 101 | 102 | // GetEntries get entries from the git index file 103 | // https://github.com/git/git/blob/master/Documentation/technical/index-format.txt 104 | func (s *Scraper) GetEntries(target *url.URL) ([]*IdxEntry, error) { 105 | idx, err := s.getIndexFile(target) 106 | if err != nil { 107 | return nil, err 108 | } 109 | var ( 110 | entryStartByteOffset = 12 111 | idxEntries = binary.BigEndian.Uint32(idx[8:entryStartByteOffset]) 112 | entryBytePtr = 12 113 | entryByteOffsetToSha = 40 114 | shaLen = 20 115 | ) 116 | var r = make([]*IdxEntry, idxEntries) 117 | for i := 0; i < int(idxEntries); i++ { 118 | var ( 119 | startOfShaOffset = entryBytePtr + entryByteOffsetToSha 120 | endOfShaOffset = startOfShaOffset + shaLen 121 | flagIdxStart = endOfShaOffset 122 | flagIdxEnd = flagIdxStart + 2 123 | startFileIdx = flagIdxEnd 124 | sha = hex.EncodeToString(idx[startOfShaOffset:endOfShaOffset]) 125 | nullIdx = bytes.Index(idx[startFileIdx:], []byte("\000")) 126 | fileName = idx[startFileIdx : startFileIdx+nullIdx] 127 | entryLen = (startFileIdx + len(fileName)) - entryBytePtr 128 | entryByte = entryLen + (8 - (entryLen % 8)) 129 | ) 130 | entry := &IdxEntry{Sha: sha, FileName: string(fileName)} 131 | r[i] = entry 132 | entryBytePtr += entryByte 133 | } 134 | 135 | return r, nil 136 | } 137 | 138 | func (s *Scraper) error(err error) { 139 | if s.config.VeryVerbose { 140 | s.errorHandler(err) 141 | } 142 | } 143 | 144 | func (s *Scraper) getAndPersist(uri string, filePath string, onDone func()) { 145 | p := path.Dir(filePath) 146 | if p == "." { 147 | return 148 | } 149 | res, err := s.client.Get(uri) 150 | defer func() { 151 | onDone() 152 | }() 153 | if err != nil { 154 | s.error(err) 155 | return 156 | } 157 | if res.StatusCode != http.StatusOK { 158 | s.error(fmt.Errorf("%s : %s", res.Status, filePath)) 159 | return 160 | } 161 | objF, err := ioutil.ReadAll(res.Body) 162 | defer res.Body.Close() 163 | if err != nil { 164 | s.error(fmt.Errorf("failed to read body on %s : %v", filePath, err)) 165 | return 166 | } 167 | r, err := zlib.NewReader(bytes.NewReader(objF)) 168 | if err != nil { 169 | s.error(fmt.Errorf("failed to create zlib reader: %v", err)) 170 | return 171 | } 172 | b, err := ioutil.ReadAll(r) 173 | if err != nil { 174 | s.error(fmt.Errorf("failed to read from zlib reader: %v", err)) 175 | return 176 | } 177 | if err := os.MkdirAll(p, os.ModePerm); err != nil { 178 | s.error(fmt.Errorf("failed to create target folder: %v", err)) 179 | return 180 | } 181 | nullIdx := bytes.Index(b, []byte("\000")) 182 | err = ioutil.WriteFile(filePath, b[nullIdx:], os.ModePerm) 183 | if err != nil { 184 | s.error(fmt.Errorf("failed to write target source: %v", err)) 185 | } 186 | } 187 | --------------------------------------------------------------------------------