7 | 
8 | 
9 |
6 |
75 |
76 | ## Todo
77 |
78 | - fix the output bug when the file is only 1 line (strpos)
79 | - change the order of the extra searches ?
80 | - ?
81 |
82 | ## Changelog
83 |
84 | **25/09/2020**
85 | - quick mode added
86 | - tokens can be read from any file
87 |
88 | **23/09/2020**
89 | - fixed an issue in the api call (params name)
90 | - added binary
91 |
92 | **13/08/2020**
93 | - fixed some types & output bugs
94 |
95 | **06/08/2020**
96 | - disabled languages and noise searches
97 | - added an option to display urls only
98 | - added an option to display only the matched parts
99 | - added an option to force the regexp to be case insensitive
100 | - creation
101 |
102 | ---
103 |
104 | Feel free to [open an issue](/../../issues/) if you have any problem with the script.
105 |
106 |
--------------------------------------------------------------------------------
/VERSION.md:
--------------------------------------------------------------------------------
1 | 1.2.1
2 |
--------------------------------------------------------------------------------
/github-regexp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gwen001/github-regexp/ea6089e76abeae45501b13e1adb60fd0919b8d3a/github-regexp
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/gwen001/github-regexp
2 |
3 | go 1.13
4 |
5 | require (
6 | github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 // indirect
7 | github.com/jpillora/go-tld v1.0.0
8 | github.com/json-iterator/go v1.1.10
9 | github.com/logrusorgru/aurora v2.0.3+incompatible
10 | golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
11 | )
12 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ=
2 | github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck=
3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
6 | github.com/jpillora/go-tld v1.0.0 h1:W0Wz3fYT9WCDNJXcXc58uV7sriLnVeELeOU5MP5X42M=
7 | github.com/jpillora/go-tld v1.0.0/go.mod h1:kitBxOF//DR5FxYeIGw+etdiiTIq5S7bx0dwy1GUNAk=
8 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
9 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
10 | github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
11 | github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
12 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
13 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
14 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
15 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
17 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
18 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
19 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
20 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
21 | golang.org/x/net v0.0.0-20180811021610-c39426892332 h1:efGso+ep0DjyCBJPjvoz0HI6UldX4Md2F1rZFe1ir0E=
22 | golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
23 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
24 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
25 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
26 | golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
27 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
28 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
29 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
30 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
31 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
32 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
33 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "fmt"
6 | "math"
7 | "math/rand"
8 | "sort"
9 | "time"
10 | "flag"
11 | "sync"
12 | "regexp"
13 | "strings"
14 | "net/url"
15 | "net/http"
16 | "io/ioutil"
17 | "crypto/md5"
18 | "encoding/hex"
19 | "encoding/json"
20 | "github.com/logrusorgru/aurora"
21 | )
22 |
23 | type Token struct {
24 | datoken string
25 | disabled_ts int64
26 | }
27 |
28 | type Search struct {
29 | signature string
30 | keyword string
31 | sort string
32 | order string
33 | language string
34 | noise []string
35 | TotalCount int
36 | }
37 |
38 | type Config struct {
39 | stop_notoken bool
40 | quick_mode bool
41 | output_mode int
42 | domain string
43 | output string
44 | tokens []Token
45 | extend bool
46 | search string
47 | delay time.Duration
48 | regexp *regexp.Regexp
49 | n_context int
50 | }
51 |
52 | type item struct {
53 | HtmlUrl string `json:"html_url"`
54 | }
55 |
56 | type response struct {
57 | Message string `json:"message"`
58 | DocumentationUrl string `json:"documentation_url"`
59 | TotalCount int `json:"total_count"`
60 | Items []item `json:"items"`
61 | }
62 |
63 | var au = aurora.NewAurora(true)
64 | var config = Config{}
65 | var t_history_urls []string
66 | var t_search []Search
67 | var t_languages = []string{"JavaScript","Python","Java","Go","Ruby","PHP","Shell","CSV","Markdown","XML","JSON","Text","CSS","HTML","Perl","ActionScript","Lua","C","C%2B%2B","C%23"}
68 | var t_noise = []string{"api","private","secret","internal","corp","development","production"}
69 | // var default_regexp = "(ConsumerKey|ConsumerSecret|DB_USERNAME|HEROKU_API_KEY|HOMEBREW_GITHUB_API_TOKEN|JEKYLL_GITHUB_TOKEN|PT_TOKEN|SESSION_TOKEN|SF_USERNAME|SLACK_BOT_TOKEN|access-token|access_token|access_token_secret|accesstoken|api-key|api_key|api_secret_key|api_token|auth_token|authkey|authorization|authorization_key|authorization_token|authtoken|aws_access_key_id|aws_secret_access_key|bearer|bot_access_token|bucket|client-secret|client_id|client_key|client_secret|clientsecret|consumer_key|consumer_secret|dbpasswd|encryption-key|encryption_key|encryptionkey|id_dsa|irc_pass|key|oauth_token|pass|password|private_key|private_key|privatekey|secret|secret-key|secret_key|secret_token|secretkey|secretkey|session_key|session_secret|slack_api_token|slack_secret_token|slack_token|ssh-key|ssh_key|sshkey|token|username|xoxa-2|xoxr|private-key)\\s*[:=>]\\s*"
70 | var default_regexp = "(access_key|access_secret|accesstoken|access_token|access-token|access_token_secret|\\.amazonaws\\.com|apikey|api_key|api-key|api_secret_key|api_token|app_key|auth|authkey|authorization|authorization_key|authorization_token|authtoken|auth_token|aws_access_key_id|aws_secret_access_key|azurewebsites|bearer|BEGIN EC PRIVATE KEY|BEGIN PGP PRIVATE KEY BLOCK|BEGIN PRIVATE KEY|BEGIN RSA PRIVATE KEY|bot_access_token|bucket|client_id|client_key|clientsecret|client_secret|client-secret|cloudapp|cloudfront|ConsumerKey|consumer_key|ConsumerSecret|consumer_secret|customer_secret|dbpasswd|DB_PASSWORD|DB_USERNAME|encryptionkey|encryption_key|encryption-key|fb_secret|firebaseio|gsecr|HEROKU_API_KEY|herokuapp|HOMEBREW_GITHUB_API_TOKEN|id_dsa|id_rsa|irc_pass|JEKYLL_GITHUB_TOKEN|key|npm_token|oauth_token|pass|password|perm|private|privatekey|secret|private_key|private-key|PT_TOKEN|rk_live_|secretkey|secret_key|secret-key|secret_token|session_key|session_secret|SESSION_TOKEN|SF_USERNAME|sk_live_|slack_api_token|SLACK_BOT_TOKEN|slack_secret_token|slack_token|sq0atp|sq0csp|sshkey|ssh_key|ssh-key|token|trafficmanager|username|user_secret|xoxa-2|xoxb-|xoxr)[a-zA-Z0-9_\\-\\.]{0,20}['\\\"]?\\s*[,:=\\[\\({]+"
71 |
72 |
73 | func parseToken( token string ) {
74 |
75 | if token == "" {
76 | token = os.Getenv("GITHUB_TOKEN")
77 | if token == "" {
78 | token = readTokenFromFile(".tokens")
79 | if token == "" {
80 | flag.Usage()
81 | fmt.Printf("\ntoken not found\n")
82 | os.Exit(-1)
83 | }
84 | }
85 | } else {
86 | if _, err := os.Stat(token); os.IsNotExist(err) {
87 | // path/to/whatever does not exist
88 | } else {
89 | token = readTokenFromFile( token )
90 | }
91 | }
92 |
93 | var t_tokens = strings.Split(token, ",")
94 | var re = regexp.MustCompile(`[0-9a-f]{40}|ghp_[a-zA-Z0-9]{36}|github_pat_[_a-zA-Z0-9]{82}`)
95 |
96 | for _,t := range t_tokens {
97 | if re.MatchString(t) {
98 | config.tokens = append( config.tokens, Token{datoken:t,disabled_ts:0} )
99 | }
100 | }
101 |
102 | rand.Seed(time.Now().UnixNano())
103 | rand.Shuffle(len(config.tokens), func(i, j int) { config.tokens[i], config.tokens[j] = config.tokens[j], config.tokens[i] })
104 |
105 | // for _,t := range config.tokens {
106 | // fmt.Println(t)
107 | // }
108 | }
109 |
110 | func readTokenFromFile( tokenfile string ) string {
111 |
112 | b, err := ioutil.ReadFile( tokenfile )
113 |
114 | if err != nil {
115 | return ""
116 | }
117 |
118 | var t_token []string
119 |
120 | for _,l := range strings.Split(string(b), "\n") {
121 | l = strings.TrimSpace( l )
122 | if len(l) > 0 && !inArray(l,t_token) {
123 | t_token = append(t_token, l)
124 | }
125 | }
126 |
127 | return strings.Join(t_token, ",")
128 | }
129 |
130 |
131 | func loadLanguages(filename string) bool {
132 |
133 | t_languages = nil
134 |
135 | if filename == "none" {
136 | return true
137 | }
138 |
139 | b, err := ioutil.ReadFile(filename)
140 |
141 | if err != nil {
142 | PrintInfos( "error", fmt.Sprintf("can't open language file: %s",filename) )
143 | os.Exit(-1)
144 | }
145 |
146 | for _,l := range strings.Split(string(b), "\n") {
147 | l = strings.TrimSpace( l )
148 | if len(l) > 0 && !inArray(l,t_languages) {
149 | t_languages = append(t_languages, l)
150 | }
151 | }
152 |
153 | return true
154 | }
155 |
156 |
157 | func loadNoise(filename string) bool {
158 |
159 | t_noise = nil
160 |
161 | if filename == "none" {
162 | return true
163 | }
164 |
165 | b, err := ioutil.ReadFile(filename)
166 |
167 | if err != nil {
168 | PrintInfos( "error", fmt.Sprintf("can't open noise file: %s",filename) )
169 | os.Exit(-1)
170 | }
171 |
172 | for _,l := range strings.Split(string(b), "\n") {
173 | l = strings.TrimSpace( l )
174 | if len(l) > 0 && !inArray(l,t_noise) {
175 | t_noise = append(t_noise, l)
176 | }
177 | }
178 |
179 | return true
180 | }
181 |
182 |
183 | func githubSearch(token string, current_search Search, page int) response {
184 |
185 | defer func() {
186 | if r := recover(); r != nil {
187 | // fmt.Println("Recovered in f", r)
188 | }
189 | }()
190 |
191 | var search = current_search.keyword
192 |
193 | if len(current_search.language) > 0 {
194 | search = fmt.Sprintf("%s+language:%s", search, current_search.language)
195 | }
196 |
197 | if len(current_search.noise) > 0 {
198 | search = fmt.Sprintf("%s+%s", search, strings.Join(current_search.noise,"+"))
199 | }
200 |
201 | // var url = fmt.Sprintf("https://api.github.com/search/code?per_page=100&sort=%s&order=%s&q=%s&page=%d", current_search.sort, current_search.order, search, page )
202 | var url = fmt.Sprintf("https://api.github.com/search/code?per_page=100&s=%s&type=Code&o=%s&q=%s&page=%d", current_search.sort, current_search.order, search, page )
203 | PrintInfos( "debug", url )
204 |
205 | client := http.Client{ Timeout: time.Second * 5 }
206 |
207 | req, err := http.NewRequest("GET", url, nil)
208 | if err != nil {
209 | PrintInfos( "error", fmt.Sprintf("%s",err) )
210 | }
211 |
212 | req.Header.Set("Authorization", "token "+token)
213 |
214 | res, getErr := client.Do(req)
215 | if getErr != nil {
216 | PrintInfos( "error", fmt.Sprintf("%s",getErr) )
217 | }
218 |
219 | if res.Body != nil {
220 | defer res.Body.Close()
221 | }
222 |
223 | body, readErr := ioutil.ReadAll(res.Body)
224 | if readErr != nil {
225 | PrintInfos( "error", fmt.Sprintf("%s",readErr) )
226 | }
227 |
228 | r := response{}
229 | jsonErr := json.Unmarshal(body, &r)
230 | if jsonErr != nil {
231 | PrintInfos( "error", fmt.Sprintf("%s",jsonErr) )
232 | }
233 |
234 | return r
235 | }
236 |
237 |
238 | func getCode( i item ) string {
239 |
240 | defer func() {
241 | if r := recover(); r != nil {
242 | // fmt.Println("Recovered in f", r)
243 | }
244 | }()
245 |
246 | var raw_url = getRawUrl(i.HtmlUrl)
247 |
248 | client := http.Client{ Timeout: time.Second * 5 }
249 |
250 | req, err := http.NewRequest("GET", raw_url, nil)
251 | if err != nil {
252 | PrintInfos( "error", fmt.Sprintf("%s",err) )
253 | }
254 |
255 | res, getErr := client.Do(req)
256 | if getErr != nil {
257 | PrintInfos( "error", fmt.Sprintf("%s",getErr) )
258 | }
259 |
260 | if res.Body != nil {
261 | defer res.Body.Close()
262 | }
263 |
264 | body, readErr := ioutil.ReadAll(res.Body)
265 | if readErr != nil {
266 | PrintInfos( "error", fmt.Sprintf("%s",readErr) )
267 | }
268 |
269 | return string(body)
270 | }
271 |
272 |
273 | func doItem(i item) {
274 |
275 | var t_match [][]int
276 |
277 | if inArray(i.HtmlUrl,t_history_urls) {
278 | // PrintInfos( "debug", fmt.Sprintf("url already checked: %s",i.HtmlUrl) )
279 | } else {
280 |
281 | t_history_urls = append(t_history_urls, i.HtmlUrl)
282 |
283 | var code = getCode( i )
284 | t_match = performRegexp( code, config.regexp )
285 |
286 | if len(t_match) > 0 {
287 |
288 | var output string
289 |
290 | for k, match := range t_match {
291 |
292 | if k > 3 {
293 | break
294 | }
295 |
296 | if k == 0 {
297 | if config.output_mode == 0 {
298 | var tmp = fmt.Sprintf("[%s] %s", time.Now().Format("15:04:05"), i.HtmlUrl )
299 | output = output + fmt.Sprintf("%s\n",au.Yellow(tmp).Bold())
300 | } else if config.output_mode == 1 {
301 | output = i.HtmlUrl + "\n"
302 | }
303 | // PrintInfos( "info", i.HtmlUrl )
304 | } else {
305 | if config.output_mode == 0 {
306 | output = output + "---\n"
307 | }
308 | }
309 |
310 | if config.output_mode != 1 {
311 | output = output + printMatch( code, match )
312 | }
313 | }
314 |
315 | fmt.Print( output )
316 | }
317 | }
318 | }
319 |
320 |
321 | func printMatch( code string, match []int ) string {
322 |
323 | if config.output_mode == 2 {
324 | return code[match[0]:match[1]]+"\n"
325 | }
326 |
327 | var prefix = get_prefix( code, match[0], config.n_context+1 );
328 | // fmt.Printf( ">>>%s<<<", prefix )
329 | var suffix = get_suffix( code, match[1], config.n_context );
330 | // fmt.Printf( ">>>%s<<<", suffix )
331 |
332 | var e = match[0]-len(prefix)
333 | // fmt.Printf( ">>>%d %d %d<<<\n", match[0], len(prefix), e )
334 | var start_line = strings.Count( code[0:e], "\n" ) + 1;
335 | // fmt.Printf( ">>>%s<<<", start_line )
336 | var final_string = fmt.Sprintf( "%s%s%s", format_string(prefix), au.BrightGreen(format_string(code[match[0]:match[1]])), format_string(suffix) )
337 | // fmt.Printf( ">>>%s<<<", final_string )
338 |
339 | var t_str = strings.Split( final_string, "\n" )
340 | var output string
341 | var padding = len(string(start_line+10)) + 1
342 |
343 | for _,str := range t_str {
344 | output = output + fmt.Sprintf( "%s%d: %s\n", strings.Repeat(" ",padding-len(string(start_line))), start_line, str )
345 | start_line++
346 | }
347 |
348 | return output
349 | }
350 |
351 |
352 | func format_string( str string ) string {
353 | if len(str) > 350 {
354 | str = str[0:350] + "..."
355 | }
356 | return str
357 | }
358 |
359 | func get_prefix( code string, pos int, n_lines int ) string {
360 | code = code[0:pos]
361 | code = Strrev( code )
362 |
363 | for i:=0 ; i