├── Gemfile
├── Gemfile.lock
├── MIT-LICENSE.txt
├── README.md
├── Rakefile
├── bin
    └── github-grep
├── github-grep.gemspec
└── lib
    └── github_grep.rb


/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | gemspec
4 | 


--------------------------------------------------------------------------------
/Gemfile.lock:
--------------------------------------------------------------------------------
 1 | PATH
 2 |   remote: .
 3 |   specs:
 4 |     github-grep (0.1.0)
 5 | 
 6 | GEM
 7 |   remote: https://rubygems.org/
 8 |   specs:
 9 |     bump (0.5.3)
10 |     rake (11.3.0)
11 | 
12 | PLATFORMS
13 |   ruby
14 | 
15 | DEPENDENCIES
16 |   bump
17 |   github-grep!
18 |   rake
19 | 
20 | BUNDLED WITH
21 |    2.1.4
22 | 


--------------------------------------------------------------------------------
/MIT-LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright (C) 2016 Michael Grosser <michael@grosser.it>
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining
 4 | a copy of this software and associated documentation files (the
 5 | "Software"), to deal in the Software without restriction, including
 6 | without limitation the rights to use, copy, modify, merge, publish,
 7 | distribute, sublicense, and/or sell copies of the Software, and to
 8 | permit persons to whom the Software is furnished to do so, subject to
 9 | the following conditions:
10 | 
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 | 
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | makes github search grep and pipeable
 2 | 
 3 | First create a [application token](https://github.com/settings/applications) with read access + enable SSO if available.
 4 | 
 5 | ```
 6 | gem install github-grep
 7 | 
 8 | export GITHUB_TOKEN=<your token>
 9 | # or: git config github.token <your token>
10 | 
11 | # search code:
12 | github-grep 'user:grosser unicorn' | grep 'dictionary' | grep -v 'higher'
13 | 
14 | # search issues and PR comments:
15 | github-grep 'repo:kubernetes/kubernetes network error' --issues | grep 'narrow-it-down' | grep -v 'something good'
16 | ```
17 | 
18 | NOTE: there are random 403 errors on the last page of a search (usually empty anyway), contacted github support about that :/
19 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | require 'bundler/setup'
2 | require 'bundler/gem_tasks'
3 | require 'bump/tasks'


--------------------------------------------------------------------------------
/bin/github-grep:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env ruby
 2 | $LOAD_PATH << File.expand_path("../../lib", __FILE__)
 3 | require "github_grep"
 4 | 
 5 | def usage
 6 |   puts <<~TEXT
 7 |     Setup
 8 |     -----
 9 |     # create a new token at https://github.com/settings/tokens/new with repo access
10 |     git config github.token NEW_TOKEN --local
11 | 
12 |     Usage
13 |     -----
14 |     #{$0} 'something to search for'
15 |   TEXT
16 |   exit 1
17 | end
18 | 
19 | 
20 | github_token = ENV['GITHUB_TOKEN'] || `git config github.token`.strip # TODO: update docs
21 | usage if github_token.empty?
22 | 
23 | type = (ARGV.delete('--issues') ? :issues : :code)
24 | 
25 | q = ARGV.shift
26 | usage if ARGV.size != 0
27 | 
28 | grep = GithubGrep.new(github_token)
29 | grep.render_search(q, type) { |slice| puts slice }
30 | 


--------------------------------------------------------------------------------
/github-grep.gemspec:
--------------------------------------------------------------------------------
 1 | require "./lib/github_grep"
 2 | 
 3 | name = "github-grep"
 4 | 
 5 | Gem::Specification.new name, GithubGrep::VERSION do |s|
 6 |   s.summary = "Makes github search grep and pipeable"
 7 |   s.authors = ["Michael Grosser"]
 8 |   s.email = "michael@grosser.it"
 9 |   s.homepage = "https://github.com/grosser/#{name}"
10 |   s.files = `git ls-files bin lib MIT-LICENSE.txt README.md`.split("\n")
11 |   s.license = "MIT"
12 |   s.executables = ['github-grep']
13 |   s.add_development_dependency "rake"
14 |   s.add_development_dependency "bump"
15 | end
16 | 


--------------------------------------------------------------------------------
/lib/github_grep.rb:
--------------------------------------------------------------------------------
 1 | require 'cgi'
 2 | require 'json'
 3 | require 'shellwords'
 4 | require 'open3'
 5 | 
 6 | class GithubGrep
 7 |   VERSION = "0.1.0"
 8 | 
 9 |   def initialize(token)
10 |     @token = token
11 |   end
12 | 
13 |   def render_search(q, type)
14 |     search(q, type) do |items|
15 |       if type == :issues
16 |         yield issue_items_to_lines(items)
17 |       else
18 |         yield code_items_to_lines(items)
19 |       end
20 |     end
21 |   end
22 | 
23 |   def search(q, type, &block)
24 |     headers = ["-H", "Accept: application/vnd.github.v3.text-match+json"]
25 |     url = "https://api.github.com/search/#{type}?q=#{CGI.escape(q)}"
26 |     all_pages(url, per_page: 100, argv: headers, &block)
27 |   end
28 | 
29 |   private
30 | 
31 |   def code_items_to_lines(items)
32 |     items.flat_map do |item|
33 |       file = item.fetch('repository').fetch('name') + ":" + item.fetch('path')
34 |       lines(item).map { |l| "#{file}: #{l}" }
35 |     end
36 |   end
37 | 
38 |   def issue_items_to_lines(items)
39 |     items.flat_map do |item|
40 |       number = item.fetch("number")
41 |       lines(item).map { |l| "##{number}: #{l}" }
42 |     end
43 |   end
44 | 
45 |   def lines(item)
46 |     item.fetch("text_matches").flat_map { |match| match.fetch('fragment').split("\n") }
47 |   end
48 | 
49 |   def all_pages(url, per_page:, **kwargs)
50 |     page = 1
51 |     connector = (url.include?("?") ? "&" : "?")
52 |     loop do
53 |       response = request_json("#{url}#{connector}per_page=#{per_page}&page=#{page}", **kwargs)
54 |       hash = response.is_a?(Hash)
55 |       if page == 1 && hash && total = response["total_count"]
56 |         $stderr.puts "Found #{total}"
57 |       else
58 |         $stderr.puts "Page #{page}"
59 |       end
60 | 
61 |       items = (hash ? response.fetch('items') : response)
62 |       yield items
63 | 
64 |       break if items.size < per_page
65 |       page += 1
66 |     end
67 |   end
68 | 
69 |   def request_json(url, argv: [])
70 |     # NOTE: github returns a 403 with a Retry-After: 60 on page 3+ ... talking with support atm but might have to handle it
71 |     command = ["curl", "-sSfv", "-H", "Authorization: token #{@token}", *argv, url]
72 | 
73 |     out, err, status = Open3.capture3(*command)
74 | 
75 |     # 403 Abuse rate limit often has no Retry-After
76 |     retry_after = err[/Retry-After: (\d+)/, 1]
77 |     abuse_limit = err.include?("returned error: 403")
78 |     if retry_after || abuse_limit
79 |       retry_after ||= "20"
80 |       warn "Sleeping #{retry_after} to avoid abuse rate-limit"
81 |       sleep Integer(retry_after)
82 |       out, err, status = Open3.capture3(*command)
83 |     end
84 | 
85 |     raise "ERROR Request failed\n#{url}\n#{err}\n#{out}" unless status.success?
86 | 
87 |     JSON.parse(out)
88 |   end
89 | end
90 | 


--------------------------------------------------------------------------------