├── .gitignore ├── LICENSE ├── README.md ├── examples ├── aws-keys.json ├── base64.json ├── cors.json ├── debug-pages.json ├── firebase.json ├── fw.json ├── go-functions.json ├── http-auth.json ├── ip.json ├── json-sec.json ├── meg-headers.json ├── php-curl.json ├── php-errors.json ├── php-serialized.json ├── php-sinks.json ├── php-sources.json ├── s3-buckets.json ├── sec.json ├── servers.json ├── strings.json ├── takeovers.json ├── upload-fields.json └── urls.json ├── gf-completion.bash ├── gf-completion.fish ├── gf-completion.zsh └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | main 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tom Hudson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gf 2 | 3 | A wrapper around grep to avoid typing common patterns. 4 | 5 | ## What? Why? 6 | 7 | I use grep a *lot*. When auditing code bases, looking at the output of [meg](https://github.com/tomnomnom/meg), 8 | or just generally dealing with large amounts of data. I often end up using fairly complex patterns like this one: 9 | 10 | ``` 11 | ▶ grep -HnrE '(\$_(POST|GET|COOKIE|REQUEST|SERVER|FILES)|php://(input|stdin))' * 12 | ``` 13 | 14 | It's really easy to mess up when typing all of that, and it can be hard to know if you haven't got any 15 | results because there are non to find, or because you screwed up writing the pattern or chose the wrong flags. 16 | 17 | I wrote `gf` to give names to the pattern and flag combinations I use all the time. So the above command 18 | becomes simply: 19 | 20 | ``` 21 | ▶ gf php-sources 22 | ``` 23 | 24 | ### Pattern Files 25 | 26 | The pattern definitions are stored in `~/.gf` as little JSON files that can be kept under version control: 27 | 28 | ``` 29 | ▶ cat ~/.gf/php-sources.json 30 | { 31 | "flags": "-HnrE", 32 | "pattern": "(\\$_(POST|GET|COOKIE|REQUEST|SERVER|FILES)|php://(input|stdin))" 33 | } 34 | ``` 35 | 36 | To help reduce pattern length and complexity a little, you can specify a list of multiple patterns too: 37 | 38 | ``` 39 | ▶ cat ~/.gf/php-sources-multiple.json 40 | { 41 | "flags": "-HnrE", 42 | "patterns": [ 43 | "\\$_(POST|GET|COOKIE|REQUEST|SERVER|FILES)", 44 | "php://(input|stdin)" 45 | ] 46 | } 47 | ``` 48 | 49 | There are some more example pattern files in the `examples` directory. 50 | 51 | You can use the `-save` flag to create pattern files from the command line: 52 | 53 | ``` 54 | ▶ gf -save php-serialized -HnrE '(a:[0-9]+:{|O:[0-9]+:"|s:[0-9]+:")' 55 | ``` 56 | 57 | ### Auto Complete 58 | 59 | There's an auto-complete script included, so you can hit 'tab' to show you what your options are: 60 | 61 | ``` 62 | ▶ gf 63 | base64 debug-pages fw php-curl php-errors php-sinks php-sources sec takeovers urls 64 | ``` 65 | 66 | #### Bash 67 | 68 | To get auto-complete working you need to `source` the `gf-completion.bash` file in your `.bashrc` or similar: 69 | 70 | ``` 71 | source ~/path/to/gf-completion.bash 72 | ``` 73 | 74 | #### Zsh 75 | 76 | To get auto-complete working you need to enable autocomplete (not needed if you have oh-my-zsh) using `autoload -U compaudit && compinit` or by putting it into `.zshrc` 77 | 78 | Then `source` the `gf-completion.zsh` file in your `.zshrc` or similar: 79 | 80 | ``` 81 | source ~/path/to/gf-completion.zsh 82 | ``` 83 | 84 | Note: if you're using oh-my-zsh or similar you may find that `gf` is an alias for `git fetch`. You can either 85 | alias the gf binary to something else, or `unalias gf` to remove the `git fetch` alias. 86 | 87 | ### Using custom engines 88 | 89 | There are some amazing code searching engines out there that can be a better replacement for grep. 90 | A good example is [the silver searcher](https://github.com/ggreer/the_silver_searcher). 91 | It's faster (like **way faster**) and presents the results in a more visually digestible manner. 92 | In order to utilize a different engine, add `engine: ` to the relevant pattern file: 93 | ```bash 94 | # Using the silver searcher instead of grep for the aws-keys pattern: 95 | # 1. Adding "ag" engine 96 | # 2. Removing the E flag which is irrelevant for ag 97 | { 98 | "engine": "ag", 99 | "flags": "-Hanr", 100 | "pattern": "([^A-Z0-9]|^)(AKIA|A3T|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{12,}" 101 | } 102 | ``` 103 | * Note: Different engines use different flags, so in the example above, the flag `E` has to be removed from the `aws-keys.json` file in order for ag to successfully run. 104 | 105 | 106 | ## Install 107 | 108 | If you've got Go installed and configured you can install `gf` with: 109 | 110 | ``` 111 | ▶ go get -u github.com/tomnomnom/gf 112 | ``` 113 | 114 | If you've installed using `go get`, you can enable auto-completion to your `.bashrc` like this: 115 | 116 | ``` 117 | ▶ echo 'source $GOPATH/src/github.com/tomnomnom/gf/gf-completion.bash' >> ~/.bashrc 118 | ``` 119 | 120 | Note that you'll have to restart your terminal, or run `source ~/.bashrc` for the changes to 121 | take effect. 122 | 123 | To get started quickly, you can copy the example pattern files to `~/.gf` like this: 124 | 125 | ``` 126 | ▶ cp -r $GOPATH/src/github.com/tomnomnom/gf/examples ~/.gf 127 | ``` 128 | 129 | My personal patterns that I've included as examples might not be very useful to you, but hopefully 130 | they're still a reasonable point of reference. 131 | 132 | ## Contributing 133 | 134 | I'd actually be most interested in new pattern files! If you've got something you regularly grep for 135 | then feel free to issue a PR to add new pattern files to the examples directory. 136 | 137 | Bug fixes are also welcome as always :) 138 | -------------------------------------------------------------------------------- /examples/aws-keys.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HanrE", 3 | "pattern": "([^A-Z0-9]|^)(AKIA|A3T|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{12,}" 4 | } 5 | -------------------------------------------------------------------------------- /examples/base64.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnroE", 3 | "pattern": "([^A-Za-z0-9+/]|^)(eyJ|YTo|Tzo|PD[89]|aHR0cHM6L|aHR0cDo|rO0)[%a-zA-Z0-9+/]+={0,2}" 4 | } 5 | 6 | 7 | -------------------------------------------------------------------------------- /examples/cors.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "patterns": [ 4 | "Access-Control-Allow" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /examples/debug-pages.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnraiE", 3 | "pattern": "(Application-Trace|Routing Error|DEBUG\"? ?[=:] ?True|Caused by:|stack trace:|Microsoft .NET Framework|Traceback|[0-9]:in `|#!/us|WebApplicationException|java\\.lang\\.|phpinfo|swaggerUi|on line [0-9]|SQLSTATE)" 4 | 5 | } 6 | -------------------------------------------------------------------------------- /examples/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-Hnri", 3 | "pattern": "firebaseio.com" 4 | } 5 | -------------------------------------------------------------------------------- /examples/fw.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "patterns": [ 4 | "django", 5 | "laravel", 6 | "symfony", 7 | "graphite", 8 | "grafana", 9 | "X-Drupal-Cache", 10 | "struts", 11 | "code ?igniter", 12 | "cake ?php", 13 | "grails", 14 | "elastic ?search", 15 | "kibana", 16 | "log ?stash", 17 | "tomcat", 18 | "jenkins", 19 | "hudson", 20 | "com.atlassian.jira", 21 | "Apache Subversion", 22 | "Chef Server", 23 | "RabbitMQ Management", 24 | "Mongo", 25 | "Travis CI - Enterprise", 26 | "BMC Remedy", 27 | "artifactory" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /examples/go-functions.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "pattern": "func [a-z0-9_]+\\(" 4 | } 5 | -------------------------------------------------------------------------------- /examples/http-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-hrioaE", 3 | "pattern": "[a-z0-9_/\\.:-]+@[a-z0-9-]+\\.[a-z0-9.-]+" 4 | } 5 | -------------------------------------------------------------------------------- /examples/ip.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnroE", 3 | "pattern": "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" 4 | } 5 | -------------------------------------------------------------------------------- /examples/json-sec.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-harioE", 3 | "pattern": "(\\\\?\"|"|%22)[a-z0-9_-]*(api[_-]?key|S3|aws_|secret|passw|auth)[a-z0-9_-]*(\\\\?\"|"|%22): ?(\\\\?\"|"|%22)[^\"&]+(\\\\?\"|"|%22)" 4 | } 5 | -------------------------------------------------------------------------------- /examples/meg-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-hroiE", 3 | "pattern": "^\u003c [a-z0-9_\\-]+: .*" 4 | } 5 | -------------------------------------------------------------------------------- /examples/php-curl.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnrE", 3 | "pattern": "CURLOPT_(HTTPHEADER|HEADER|COOKIE|RANGE|REFERER|USERAGENT|PROXYHEADER)" 4 | } 5 | -------------------------------------------------------------------------------- /examples/php-errors.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "patterns": [ 4 | "php warning", 5 | "php error", 6 | "fatal error", 7 | "uncaught exception", 8 | "include_path", 9 | "undefined index", 10 | "undefined variable", 11 | "\\?php", 12 | "<\\?[^x]", 13 | "stack trace\\:", 14 | "expects parameter [0-9]*", 15 | "Debug Trace" 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /examples/php-serialized.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnrE", 3 | "patterns": [ 4 | "a:[0-9]+:{", 5 | "O:[0-9]+:\"", 6 | "s:[0-9]+:\"" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /examples/php-sinks.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "pattern": "[^a-z0-9_](system|exec|popen|pcntl_exec|eval|create_function|unserialize|file_exists|md5_file|filemtime|filesize|assert) ?\\(" 4 | } 5 | -------------------------------------------------------------------------------- /examples/php-sources.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnrE", 3 | "patterns": [ 4 | "\\$_(POST|GET|COOKIE|REQUEST|SERVER|FILES)", 5 | "php://(input|stdin)" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /examples/s3-buckets.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-hrioaE", 3 | "patterns": [ 4 | "[a-z0-9.-]+\\.s3\\.amazonaws\\.com", 5 | "[a-z0-9.-]+\\.s3-[a-z0-9-]\\.amazonaws\\.com", 6 | "[a-z0-9.-]+\\.s3-website[.-](eu|ap|us|ca|sa|cn)", 7 | "//s3\\.amazonaws\\.com/[a-z0-9._-]+", 8 | "//s3-[a-z0-9-]+\\.amazonaws\\.com/[a-z0-9._-]+" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /examples/sec.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HanriE", 3 | "pattern": "(aws_access|aws_secret|api[_-]?key|ListBucketResult|S3_ACCESS_KEY|Authorization:|RSA PRIVATE|Index of|aws_|secret|ssh-rsa AA)" 4 | } 5 | -------------------------------------------------------------------------------- /examples/servers.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-hri", 3 | "pattern": "server: " 4 | } 5 | -------------------------------------------------------------------------------- /examples/strings.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-hroiaE", 3 | "patterns": [ 4 | "\"[^\"]+\"", 5 | "'[^']+'" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /examples/takeovers.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "patterns": [ 4 | "There is no app configured at that hostname", 5 | "NoSuchBucket", 6 | "No Such Account", 7 | "You're Almost There", 8 | "a GitHub Pages site here", 9 | "There's nothing here", 10 | "project not found", 11 | "Your CNAME settings", 12 | "InvalidBucketName", 13 | "PermanentRedirect", 14 | "The specified bucket does not exist", 15 | "Repository not found", 16 | "Sorry, We Couldn't Find That Page", 17 | "The feed has not been found.", 18 | "The thing you were looking for is no longer here, or never was", 19 | "Please renew your subscription", 20 | "There isn't a Github Pages site here.", 21 | "We could not find what you're looking for.", 22 | "No settings were found for this company:", 23 | "No such app", 24 | "is not a registered InCloud YouTrack", 25 | "Unrecognized domain", 26 | "project not found", 27 | "This UserVoice subdomain is currently available!", 28 | "Do you want to register", 29 | "Help Center Closed" 30 | ] 31 | } 32 | 33 | -------------------------------------------------------------------------------- /examples/upload-fields.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-HnriE", 3 | "pattern": "\u003cinput[^\u003e]+type=[\"']?file[\"']?" 4 | } 5 | -------------------------------------------------------------------------------- /examples/urls.json: -------------------------------------------------------------------------------- 1 | { 2 | "flags": "-oriahE", 3 | "pattern": "https?://[^\"\\'> ]+" 4 | } 5 | -------------------------------------------------------------------------------- /gf-completion.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | complete -W "\$(gf -list)" gf 3 | -------------------------------------------------------------------------------- /gf-completion.fish: -------------------------------------------------------------------------------- 1 | complete -x -c gf -a "(gf -list)" 2 | -------------------------------------------------------------------------------- /gf-completion.zsh: -------------------------------------------------------------------------------- 1 | compdef _gf gf 2 | 3 | function _gf { 4 | _arguments "1: :($(gf -list))" 5 | } 6 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "os/user" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | type pattern struct { 16 | Flags string `json:"flags,omitempty"` 17 | Pattern string `json:"pattern,omitempty"` 18 | Patterns []string `json:"patterns,omitempty"` 19 | Engine string `json:"engine,omitempty"` 20 | } 21 | 22 | func main() { 23 | var saveMode bool 24 | flag.BoolVar(&saveMode, "save", false, "save a pattern (e.g: gf -save pat-name -Hnri 'search-pattern')") 25 | 26 | var listMode bool 27 | flag.BoolVar(&listMode, "list", false, "list available patterns") 28 | 29 | var dumpMode bool 30 | flag.BoolVar(&dumpMode, "dump", false, "prints the grep command rather than executing it") 31 | 32 | flag.Parse() 33 | 34 | if listMode { 35 | pats, err := getPatterns() 36 | if err != nil { 37 | fmt.Fprintf(os.Stderr, "%s\n", err) 38 | return 39 | } 40 | 41 | fmt.Println(strings.Join(pats, "\n")) 42 | return 43 | } 44 | 45 | if saveMode { 46 | name := flag.Arg(0) 47 | flags := flag.Arg(1) 48 | pattern := flag.Arg(2) 49 | 50 | err := savePattern(name, flags, pattern) 51 | if err != nil { 52 | fmt.Fprintf(os.Stderr, "%s\n", err) 53 | } 54 | return 55 | } 56 | 57 | patName := flag.Arg(0) 58 | files := flag.Arg(1) 59 | if files == "" { 60 | files = "." 61 | } 62 | 63 | patDir, err := getPatternDir() 64 | if err != nil { 65 | fmt.Fprintln(os.Stderr, "unable to open user's pattern directory") 66 | return 67 | } 68 | 69 | filename := filepath.Join(patDir, patName+".json") 70 | f, err := os.Open(filename) 71 | if err != nil { 72 | fmt.Fprintln(os.Stderr, "no such pattern") 73 | return 74 | } 75 | defer f.Close() 76 | 77 | pat := pattern{} 78 | dec := json.NewDecoder(f) 79 | err = dec.Decode(&pat) 80 | 81 | if err != nil { 82 | fmt.Fprintf(os.Stderr, "pattern file '%s' is malformed: %s\n", filename, err) 83 | return 84 | } 85 | 86 | if pat.Pattern == "" { 87 | // check for multiple patterns 88 | if len(pat.Patterns) == 0 { 89 | fmt.Fprintf(os.Stderr, "pattern file '%s' contains no pattern(s)\n", filename) 90 | return 91 | } 92 | 93 | pat.Pattern = "(" + strings.Join(pat.Patterns, "|") + ")" 94 | } 95 | 96 | if dumpMode { 97 | fmt.Printf("grep %v %q %v\n", pat.Flags, pat.Pattern, files) 98 | 99 | } else { 100 | var cmd *exec.Cmd 101 | operator := "grep" 102 | if pat.Engine != "" { 103 | operator = pat.Engine 104 | } 105 | 106 | if stdinIsPipe() { 107 | cmd = exec.Command(operator, pat.Flags, pat.Pattern) 108 | } else { 109 | cmd = exec.Command(operator, pat.Flags, pat.Pattern, files) 110 | } 111 | cmd.Stdin = os.Stdin 112 | cmd.Stdout = os.Stdout 113 | cmd.Stderr = os.Stderr 114 | cmd.Run() 115 | } 116 | 117 | } 118 | 119 | func getPatternDir() (string, error) { 120 | usr, err := user.Current() 121 | if err != nil { 122 | return "", err 123 | } 124 | path := filepath.Join(usr.HomeDir, ".config/gf") 125 | if _, err := os.Stat(path); !os.IsNotExist(err) { 126 | // .config/gf exists 127 | return path, nil 128 | } 129 | return filepath.Join(usr.HomeDir, ".gf"), nil 130 | } 131 | 132 | func savePattern(name, flags, pat string) error { 133 | if name == "" { 134 | return errors.New("name cannot be empty") 135 | } 136 | 137 | if pat == "" { 138 | return errors.New("pattern cannot be empty") 139 | } 140 | 141 | p := &pattern{ 142 | Flags: flags, 143 | Pattern: pat, 144 | } 145 | 146 | patDir, err := getPatternDir() 147 | if err != nil { 148 | return fmt.Errorf("failed to determine pattern directory: %s", err) 149 | } 150 | 151 | path := filepath.Join(patDir, name+".json") 152 | f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) 153 | if err != nil { 154 | return fmt.Errorf("failed to create pattern file: %s", err) 155 | } 156 | defer f.Close() 157 | 158 | enc := json.NewEncoder(f) 159 | enc.SetIndent("", " ") 160 | 161 | err = enc.Encode(p) 162 | if err != nil { 163 | return fmt.Errorf("failed to write pattern file: %s", err) 164 | } 165 | 166 | return nil 167 | } 168 | 169 | func getPatterns() ([]string, error) { 170 | out := []string{} 171 | 172 | patDir, err := getPatternDir() 173 | if err != nil { 174 | return out, fmt.Errorf("failed to determine pattern directory: %s", err) 175 | } 176 | _ = patDir 177 | 178 | files, err := filepath.Glob(patDir + "/*.json") 179 | if err != nil { 180 | return out, err 181 | } 182 | 183 | for _, f := range files { 184 | f = f[len(patDir)+1 : len(f)-5] 185 | out = append(out, f) 186 | } 187 | 188 | return out, nil 189 | } 190 | 191 | func stdinIsPipe() bool { 192 | stat, _ := os.Stdin.Stat() 193 | return (stat.Mode() & os.ModeCharDevice) == 0 194 | } 195 | --------------------------------------------------------------------------------