├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── lib
    ├── helper.go
    └── http.go
├── main.go
└── secretz.png


/.gitignore:
--------------------------------------------------------------------------------
1 | output/
2 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: go
 2 | 
 3 | go:
 4 |   - "1.10.x"
 5 |   - "1.11.x"
 6 |   - "1.12.x"
 7 |   - "1.13.x"
 8 |   - master
 9 | 
10 | script: go vet ./... 
11 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 CORBEN LEO
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | <p align="center">
 2 | <img src="https://github.com/lc/secretz/raw/master/secretz.png" alt="secretz" width="250" />
 3 | </p>
 4 | 
 5 | # secretz
 6 | [![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
 7 | [![Build Status](https://travis-ci.org/lc/secretz.svg?branch=master)](https://travis-ci.org/lc/secretz)
 8 | [![Go ReportCard](https://goreportcard.com/badge/github.com/lc/secretz#1)](https://goreportcard.com/report/github.com/lc/secretz)
 9 | 
10 | `secretz` is a tool that minimizes the large attack surface of Travis CI. It automatically fetches repos, builds, and logs for any given organization. 
11 | 
12 | Built during and for our research on TravisCI: https://edoverflow.com/2019/ci-knew-there-would-be-bugs-here/
13 | 
14 | 
15 | ## Usage:
16 | `secretz -t Organization [options]`
17 | 
18 | 
19 | ### Flags:
20 | | Flag | Description | Example |
21 | |------|-------------|---------|
22 | | `-t` | Organization to get repos, builds, and logs for | `secretz -t ExampleCo` |
23 | | `-c` | Limit the number of workers that are spawned | `secretz -t ExampleCo -c 3` |
24 | | `-delay` | delay between requests + random delay/2 jitter | `secretz -t ExampleCo -delay 900`|
25 | | `-members [list \| scan]` | Get all GitHub members belonging to Organization and list/scan them | `secretz -t ExampleCo -members scan` |
26 | | `-timeout` | How long to wait for HTTP Responses from Travis CI | `secretz -t ExampleCo -timeout 20` |
27 | | `-setkey` | Set API Key for api.travis-ci.org | `secretz -setkey yourapikey` |
28 | 
29 | ## Installation:
30 | 
31 | ### Via `go get`
32 | ```
33 | go get -u github.com/lc/secretz
34 | ```
35 | 
36 | ### Via `git clone`
37 | 
38 | ```
39 | go get -u github.com/json-iterator/go
40 | git clone git@github.com:lc/secretz
41 | cd secretz && go build -o secretz main.go
42 | ```
43 | 
44 | 
45 | ### Generate an API-Key: 
46 | ```
47 | travis login
48 | travis token --org
49 | ```
50 | 
51 | ### Create config file
52 | `secretz -setkey <API-KEY>`
53 | 
54 | 
55 | ### Note:
56 | Please keep your delay high and your workers low out of respect for TravisCI and their APIs. This will also help you from being rate-limited by them. 
57 | 


--------------------------------------------------------------------------------
/lib/helper.go:
--------------------------------------------------------------------------------
  1 | package lib
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"fmt"
  6 | 	"io/ioutil"
  7 | 	"log"
  8 | 	"os"
  9 | 	"os/user"
 10 | )
 11 | 
 12 | var (
 13 | 	home = HomeDir()
 14 | 	path = home + "/.config/secretz/config.json"
 15 | 	c    Config
 16 | )
 17 | 
 18 | // Config struct is used to unmarshal our config file into.
 19 | type Config struct {
 20 | 	TravisCIOrg string `json:"TravisCIOrgKey"`
 21 | }
 22 | 
 23 | // Usage function displays the usage of the tool.
 24 | func Usage() {
 25 | 	help := `Usage: secretz -t <organization> [options]
 26 | 
 27 |   -c int
 28 | 		Number of concurrent fetchers (default 5)
 29 | 		
 30 |   -delay int
 31 | 		delay between requests + random delay/2 jitter (default 600)
 32 | 		
 33 |   -members string
 34 | 		Retrieve members of Github Org parameters: [list | scan]
 35 | 		
 36 |   -setkey string
 37 | 		Set API Key for api.travis-ci.org
 38 | 		
 39 |   -t string
 40 | 		Target organization
 41 | 		
 42 |   -timeout int
 43 | 		Timeout for the tool in seconds (default 30)`
 44 | 	fmt.Printf(help)
 45 | 
 46 | }
 47 | 
 48 | // GetAPIKey opens the config file and returns an API key
 49 | func GetAPIKey() string {
 50 | 	if Exists(path) {
 51 | 		data, err := ioutil.ReadFile(path)
 52 | 		if err != nil {
 53 | 			log.Fatalf("Error opening config file: %v", err)
 54 | 		}
 55 | 		mErr := json.Unmarshal(data, &c)
 56 | 		if mErr != nil {
 57 | 			c.TravisCIOrg = ""
 58 | 		}
 59 | 	}
 60 | 	CreateOutputDir(home + "/.config/secretz")
 61 | 	return c.TravisCIOrg
 62 | }
 63 | 
 64 | // SetAPIKey takes an API key and saves it to a config file.
 65 | func SetAPIKey(key string) {
 66 | 	c := Config{key}
 67 | 	bytes, err := json.Marshal(c)
 68 | 	if err != nil {
 69 | 		log.Fatalf("Error marshalling data: %v", err)
 70 | 	}
 71 | 	CreateOutputDir(home + "/.config/secretz/")
 72 | 	err = ioutil.WriteFile(path, bytes, 0644)
 73 | 	if err != nil {
 74 | 		log.Fatalf("Error creating config file: %v\n", err)
 75 | 	}
 76 | 	fmt.Printf("Created config file: %s\n", path)
 77 | }
 78 | 
 79 | // Exists checks if the location provided exists or not.
 80 | func Exists(loc string) bool {
 81 | 	_, err := os.Stat(loc)
 82 | 	if err == nil {
 83 | 		return true
 84 | 	}
 85 | 	if os.IsNotExist(err) {
 86 | 		return false
 87 | 	}
 88 | 	return true
 89 | 
 90 | }
 91 | 
 92 | // HomeDir returns the user's home directory.
 93 | func HomeDir() string {
 94 | 	usr, err := user.Current()
 95 | 	if err != nil {
 96 | 		log.Fatalf("Error: %v", err)
 97 | 		os.Exit(1)
 98 | 	}
 99 | 	return usr.HomeDir
100 | }
101 | 
102 | // CreateOrg creates a directory for a given Organization to hold the build logs.
103 | func CreateOrg(dir string) {
104 | 	CreateOutputDir("output")
105 | 	dir = fmt.Sprintf("output/%s", dir)
106 | 	if _, err := os.Stat(dir); os.IsNotExist(err) {
107 | 		err = os.Mkdir(dir, 0755)
108 | 		if err != nil {
109 | 			log.Fatalf("Error: %v", err)
110 | 			os.Exit(1)
111 | 		}
112 | 	}
113 | }
114 | 
115 | // CreateOutputDir creates a directory from a given path
116 | func CreateOutputDir(dir string) {
117 | 	if _, err := os.Stat(dir); os.IsNotExist(err) {
118 | 		err = os.Mkdir(dir, 0755)
119 | 		if err != nil {
120 | 			log.Fatalf("Error: %v", err)
121 | 		}
122 | 	}
123 | }
124 | 


--------------------------------------------------------------------------------
/lib/http.go:
--------------------------------------------------------------------------------
 1 | package lib
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"log"
 8 | 	"net/http"
 9 | 	"time"
10 | )
11 | 
12 | // GitHubAPI is used to unmarshal Github's API response. This holds usernames.
13 | type GitHubAPI []struct {
14 | 	Login string `json:"login"`
15 | }
16 | 
17 | var key = GetAPIKey()
18 | 
19 | // Secretz is the http client for the tool.
20 | var Secretz = &http.Client{
21 | 	Timeout: time.Second * 10,
22 | }
23 | 
24 | // OrgMembers queries GitHub's API and gets a list of members for a given org.
25 | func OrgMembers(target string) (g *GitHubAPI) {
26 | 	memStruct := new(GitHubAPI)
27 | 	target = fmt.Sprintf("https://api.github.com/orgs/%s/public_members", target)
28 | 	resp, err := Secretz.Get(target)
29 | 	if err != nil {
30 | 		log.Fatalf("Error creating HTTP Request: %v", err)
31 | 	}
32 | 	defer resp.Body.Close()
33 | 	err = json.NewDecoder(resp.Body).Decode(&memStruct)
34 | 	if err != nil {
35 | 		log.Fatalf("Error parsing response from GitHub: %v", err)
36 | 	}
37 | 	return memStruct
38 | }
39 | 
40 | // QueryApi is used to query Travis-CI's API
41 | func QueryApi(target string) (body []byte) {
42 | 	req, err := http.NewRequest("GET", target, nil)
43 | 	if err != nil {
44 | 		log.Fatalf("Error creating HTTP Request: %v", err)
45 | 	}
46 | 	req.Header.Add("User-Agent", `API Explorer`)
47 | 	if key != "" {
48 | 		token := fmt.Sprintf("token %s", key)
49 | 		req.Header.Add("Authorization", token)
50 | 	}
51 | 	req.Header.Add("Travis-API-Version", `3`)
52 | 	resp, err := Secretz.Do(req)
53 | 	if err != nil {
54 | 		log.Fatalf("Could not request API: %v", err)
55 | 	}
56 | 	if resp.StatusCode != 200 {
57 | 		log.Fatal("\nTravisCI responded with a non-200 statuscode. You're likely being rate-limited")
58 | 	}
59 | 	defer resp.Body.Close()
60 | 	bytes, err := ioutil.ReadAll(resp.Body)
61 | 	if err != nil {
62 | 		log.Fatalf("Error parsing response from TravisCI: %v", err)
63 | 	}
64 | 	return bytes
65 | }
66 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"flag"
  5 | 	"fmt"
  6 | 	"io/ioutil"
  7 | 	"log"
  8 | 	"math/rand"
  9 | 	"net/url"
 10 | 	"os"
 11 | 	"sync"
 12 | 	"time"
 13 | 
 14 | 	"github.com/json-iterator/go"
 15 | 	"github.com/lc/secretz/lib"
 16 | )
 17 | 
 18 | var repos []string
 19 | 
 20 | // TravisCI is the struct that holds repo slugs.
 21 | type TravisCI struct {
 22 | 	Repositories []struct {
 23 | 		Slug   string `json:"slug"`
 24 | 		Active bool   `json:"active"`
 25 | 	} `json:"repositories"`
 26 | }
 27 | 
 28 | // Builds is the struct that build information from TravisCI
 29 | type Builds struct {
 30 | 	Type       string `json:"@type"`
 31 | 	Pagination struct {
 32 | 		IsFirst bool `json:"is_first"`
 33 | 		IsLast  bool `json:"is_last"`
 34 | 	} `json:"@pagination"`
 35 | 	Builds []struct {
 36 | 		Jobs []struct {
 37 | 			JobId int `json:"id"`
 38 | 		} `json:"jobs"`
 39 | 	} `json:"builds"`
 40 | }
 41 | 
 42 | var (
 43 | 	setkey      string
 44 | 	org         string
 45 | 	timeout     int
 46 | 	concurrency int
 47 | 	delay       int
 48 | 	// GetMembers is a flag with the options: list or scan
 49 | 	GetMembers string
 50 | 	targets    []string
 51 | )
 52 | 
 53 | func init() {
 54 | 	flag.Usage = lib.Usage
 55 | 	flag.StringVar(&setkey, "setkey", "", "Set API Key for api.travis-ci.org")
 56 | 	flag.StringVar(&org, "t", "", "Target organization")
 57 | 	flag.IntVar(&timeout, "timeout", 30, "Timeout for the tool in seconds")
 58 | 	flag.IntVar(&concurrency, "c", 5, "Number of concurrent fetchers")
 59 | 	flag.StringVar(&GetMembers, "members", "", "Get GitHub Org Members")
 60 | 	flag.IntVar(&delay, "delay", 600, "delay between requests + random delay/2 jitter")
 61 | }
 62 | 
 63 | // Job struct holds a JobID and the Organization name.
 64 | type Job struct {
 65 | 	ID  int
 66 | 	Org string
 67 | }
 68 | 
 69 | func main() {
 70 | 	flag.Parse()
 71 | 
 72 | 	lib.Secretz.Timeout = time.Duration(timeout) * time.Second
 73 | 	if setkey != "" {
 74 | 		lib.SetAPIKey(setkey)
 75 | 	}
 76 | 	if len(org) < 1 || setkey != "" {
 77 | 		log.Fatalf("Usage: %s -t <organization>\n", os.Args[0])
 78 | 	}
 79 | 	if GetMembers == "" {
 80 | 		targets = append(targets, org)
 81 | 	} else {
 82 | 		switch GetMembers {
 83 | 		case "list":
 84 | 			GHMem := lib.OrgMembers(org)
 85 | 			for _, Member := range *GHMem {
 86 | 				fmt.Println(Member.Login)
 87 | 			}
 88 | 			os.Exit(0)
 89 | 		case "scan":
 90 | 			GHMem := lib.OrgMembers(org)
 91 | 			for _, Member := range *GHMem {
 92 | 				targets = append(targets, Member.Login)
 93 | 			}
 94 | 			break
 95 | 		default:
 96 | 			log.Fatalf("Invalid option specified in -members flag!\n")
 97 | 		}
 98 | 	}
 99 | 	for _, org := range targets {
100 | 		lib.CreateOrg(org)
101 | 		log.Printf("Fetching repos for %s\n", org)
102 | 		ParseResponse(org)
103 | 		log.Printf("Fetching builds for %s's repos\n", org)
104 | 
105 | 		jobChan := make(chan *Job)
106 | 		finishedChan := make(chan string)
107 | 		var wg, wg2 sync.WaitGroup
108 | 		wg2.Add(1)
109 | 
110 | 		for i := 0; i < concurrency; i++ {
111 | 			wg.Add(1)
112 | 			go func() {
113 | 				defer wg.Done()
114 | 				SaveLogs(jobChan, finishedChan)
115 | 			}()
116 | 		}
117 | 
118 | 		go func() {
119 | 			defer wg2.Done()
120 | 			for str := range finishedChan {
121 | 				log.Printf("%s\n", str)
122 | 			}
123 | 		}()
124 | 
125 | 		go func() {
126 | 			for _, slug := range repos {
127 | 				builds := GetBuilds(slug)
128 | 				for _, job := range builds {
129 | 					jobChan <- &Job{ID: job, Org: org}
130 | 				}
131 | 			}
132 | 			close(jobChan)
133 | 		}()
134 | 
135 | 		wg.Wait()
136 | 		close(finishedChan)
137 | 		wg2.Wait()
138 | 	}
139 | }
140 | 
141 | // ParseResponse gets the JSON response from Travis and parses it for repo slugs
142 | func ParseResponse(org string) {
143 | 	for {
144 | 		api := fmt.Sprintf("https://api.travis-ci.org/owner/%s/repos?limit=100&offset=%d", org, len(repos))
145 | 		res := lib.QueryApi(api)
146 | 
147 | 		ciResp := new(TravisCI)
148 | 		err := jsoniter.Unmarshal(res, ciResp)
149 | 		if err != nil {
150 | 			log.Fatalf("Could not decode json: %s\n", err)
151 | 		}
152 | 
153 | 		if len(ciResp.Repositories) == 0 {
154 | 			break
155 | 		}
156 | 		for _, repo := range ciResp.Repositories {
157 | 			repos = append(repos, repo.Slug)
158 | 		}
159 | 	}
160 | }
161 | 
162 | // GetBuilds gets all JobId's from builds of a repo.
163 | func GetBuilds(slug string) []int {
164 | 
165 | 	builds := new(Builds)
166 | 	jobs := []int{}
167 | 	offset := 0
168 | 
169 | 	for {
170 | 		log.Printf("Fetching builds %s [offset: %d]\n", slug, offset)
171 | 		api := fmt.Sprintf("https://api.travis-ci.org/repo/%s/builds?limit=100&offset=%d", url.QueryEscape(slug), offset)
172 | 		res := lib.QueryApi(api)
173 | 
174 | 		// delay + jitter
175 | 		if delay > 0 {
176 | 			time.Sleep((time.Duration(delay + rand.Intn(delay/2))) * time.Millisecond)
177 | 		}
178 | 
179 | 		err := jsoniter.Unmarshal(res, builds)
180 | 		if err != nil {
181 | 			log.Fatalf("Could not decode json: %s\n", err)
182 | 		}
183 | 		if builds.Type == "error" {
184 | 			break
185 | 		}
186 | 
187 | 		loop := len(builds.Builds)
188 | 		i := 0
189 | 		for i < loop {
190 | 			for _, z := range builds.Builds[i].Jobs {
191 | 				jobs = append(jobs, z.JobId)
192 | 			}
193 | 			i++
194 | 		}
195 | 
196 | 		if builds.Pagination.IsLast {
197 | 			break
198 | 		}
199 | 		offset += 100
200 | 	}
201 | 
202 | 	return jobs
203 | }
204 | 
205 | // SaveLogs saves build logs for given job ids
206 | func SaveLogs(jobChan chan *Job, resultChan chan string) {
207 | 	for job := range jobChan {
208 | 		api := fmt.Sprintf("https://api.travis-ci.org/job/%d/log.txt", job.ID)
209 | 		// delay + jitter
210 | 		if delay > 0 {
211 | 			time.Sleep((time.Duration(delay + rand.Intn(delay/2))) * time.Millisecond)
212 | 		}
213 | 		res := lib.QueryApi(api)
214 | 		file := fmt.Sprintf("output/%s/%d.txt", job.Org, job.ID)
215 | 		err := ioutil.WriteFile(file, res, 0644)
216 | 		if err != nil {
217 | 			log.Fatalf("Error: %v", err)
218 | 		}
219 | 		resultChan <- fmt.Sprintf("Wrote log %d to %s", job.ID, file)
220 | 	}
221 | }
222 | 


--------------------------------------------------------------------------------
/secretz.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lc/secretz/4d14ecd2a285085ebadfe5289fe3c38ab6c7189d/secretz.png


--------------------------------------------------------------------------------