├── Procfile ├── .gitignore ├── config.ru ├── .env-example ├── screenshot.png ├── .travis.yml ├── Gemfile ├── test_mapping.json ├── Gemfile.lock ├── server.rb ├── README.md ├── unittests.rb └── application_helper.rb /Procfile: -------------------------------------------------------------------------------- 1 | web: ruby server.rb -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | require "./server" 2 | run GHAapp 3 | -------------------------------------------------------------------------------- /.env-example: -------------------------------------------------------------------------------- 1 | GITHUB_PRIVATE_KEY="" 2 | GITHUB_APP_IDENTIFIER= 3 | GITHUB_WEBHOOK_SECRET= 4 | PORT=3000 5 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wordpress-mobile/test-case-reminder-bot/trunk/screenshot.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | 3 | script: ruby unittests.rb 4 | 5 | # whitelist branches for the "push" build check 6 | branches: 7 | only: 8 | - master -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'sinatra', '~> 2.0' 4 | gem 'jwt', '~> 2.1' 5 | gem 'octokit', '~> 4.0' 6 | gem 'dotenv' 7 | 8 | group :development do 9 | gem 'rerun' 10 | gem 'test-unit' 11 | end 12 | gem "test-unit-rr", "~> 1.0" 13 | -------------------------------------------------------------------------------- /test_mapping.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "regex":"gutenberg/packages/block-library/src/image/edit.native.js", 4 | "testFile": ["gutenberg/image-basic.md", "gutenberg/image-upload.md"] 5 | }, 6 | { 7 | "regex":"gutenberg/packages/block-library/src/video/edit.native.js", 8 | "testFile": ["gutenberg/video-sanity.md", "gutenberg/video-upload.md"] 9 | }, 10 | { 11 | "regex":"gutenberg/packages/block-editor/src/components/media-upload/index.native.js", 12 | "testFile": ["gutenberg/image-upload.md", "gutenberg/video-upload.md", "gutenberg/media-text-upload.md"] 13 | }, 14 | { 15 | "regex":".*/Gutenberg/Processors/GutenbergImgUploadProcessor.swift", 16 | "testFile": ["gutenberg/img-upload-processor.md", "gutenberg/multi-block-media-upload.md"] 17 | }, 18 | { 19 | "regex":".*/Gutenberg/Processors/GutenbergVideoUploadProcessor.swift", 20 | "testFile": ["gutenberg/video-upload-processor.md", "gutenberg/multi-block-media-upload.md"] 21 | }, 22 | { 23 | "regex":".*/Gutenberg/Processors/GutenbergGalleryUploadProcessor.swift", 24 | "testFile": ["gutenberg/gallery-upload-processor.md", "gutenberg/multi-block-media-upload.md"] 25 | } 26 | ] -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | addressable (2.5.2) 5 | public_suffix (>= 2.0.2, < 4.0) 6 | dotenv (2.5.0) 7 | faraday (0.15.3) 8 | multipart-post (>= 1.2, < 3) 9 | ffi (1.13.0) 10 | jwt (2.1.0) 11 | listen (3.2.1) 12 | rb-fsevent (~> 0.10, >= 0.10.3) 13 | rb-inotify (~> 0.9, >= 0.9.10) 14 | multipart-post (2.0.0) 15 | mustermann (1.0.3) 16 | octokit (4.13.0) 17 | sawyer (~> 0.8.0, >= 0.5.3) 18 | power_assert (1.2.0) 19 | public_suffix (3.0.3) 20 | rack (2.0.6) 21 | rack-protection (2.0.4) 22 | rack 23 | rb-fsevent (0.10.4) 24 | rb-inotify (0.10.1) 25 | ffi (~> 1.0) 26 | rerun (0.13.0) 27 | listen (~> 3.0) 28 | rr (1.2.1) 29 | sawyer (0.8.1) 30 | addressable (>= 2.3.5, < 2.6) 31 | faraday (~> 0.8, < 1.0) 32 | sinatra (2.0.4) 33 | mustermann (~> 1.0) 34 | rack (~> 2.0) 35 | rack-protection (= 2.0.4) 36 | tilt (~> 2.0) 37 | test-unit (3.3.5) 38 | power_assert 39 | test-unit-rr (1.0.5) 40 | rr (>= 1.1.1) 41 | test-unit (>= 2.5.2) 42 | tilt (2.0.8) 43 | 44 | PLATFORMS 45 | ruby 46 | 47 | DEPENDENCIES 48 | dotenv 49 | jwt (~> 2.1) 50 | octokit (~> 4.0) 51 | rerun 52 | sinatra (~> 2.0) 53 | test-unit 54 | test-unit-rr (~> 1.0) 55 | 56 | BUNDLED WITH 57 | 1.17.1 58 | -------------------------------------------------------------------------------- /server.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra' 2 | require 'octokit' 3 | require 'dotenv/load' # Manages environment variables 4 | require 'json' 5 | require 'openssl' # Verifies the webhook signature 6 | require 'jwt' # Authenticates a GitHub App 7 | require 'time' # Gets ISO 8601 representation of a Time object 8 | require 'logger' # Logs debug statements 9 | require "base64" 10 | require './application_helper.rb' 11 | 12 | set :port, ENV['PORT'] 13 | set :bind, '0.0.0.0' 14 | 15 | # This is template code to create a GitHub App server. 16 | # You can read more about GitHub Apps here: # https://developer.github.com/apps/ 17 | # 18 | # On its own, this app does absolutely nothing, except that it can be installed. 19 | # It's up to you to add functionality! 20 | # You can check out one example in advanced_server.rb. 21 | # 22 | # This code is a Sinatra app, for two reasons: 23 | # 1. Because the app will require a landing page for installation. 24 | # 2. To easily handle webhook events. 25 | # 26 | # Of course, not all apps need to receive and process events! 27 | # Feel free to rip out the event handling code if you don't need it. 28 | # 29 | # Have fun! 30 | # 31 | 32 | class GHAapp < Sinatra::Application 33 | helpers ApplicationHelper 34 | 35 | # Turn on Sinatra's verbose logging during development 36 | configure :development do 37 | set :logging, Logger::DEBUG 38 | end 39 | 40 | configure :production do 41 | set :logging, Logger::DEBUG 42 | end 43 | 44 | # Before each request to the `/event_handler` route 45 | before '/event_handler' do 46 | get_payload_request(request) 47 | verify_webhook_signature 48 | authenticate_app 49 | # Authenticate the app installation in order to run API operations 50 | authenticate_installation(@payload) 51 | end 52 | 53 | 54 | post '/event_handler' do 55 | case request.env['HTTP_X_GITHUB_EVENT'] 56 | when 'pull_request' 57 | if @payload['action'] === 'opened' 58 | handle_pullrequest_opened_event(@payload) 59 | end 60 | # if @payload['action'] === 'edited' 61 | # handle_pullrequest_opened_event(@payload) 62 | # end 63 | end 64 | 65 | 200 # success status 66 | end 67 | 68 | # Finally some logic to let us run this server directly from the command line, 69 | # or with Rack. Don't worry too much about this code. But, for the curious: 70 | # $0 is the executed file 71 | # __FILE__ is the current file 72 | # If they are the same—that is, we are running this file directly, call the 73 | # Sinatra run method 74 | run! if __FILE__ == $0 75 | end 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Test Case Reminder Bot 2 | 3 | A GitHub bot that drops a comment into the PR about what to test when specific files change. It is done by providing source file to test-suite mapping, and the test-suite files itself. 4 | 5 | ![Screenshot](screenshot.png) 6 | 7 | ## Configuration 8 | 9 | By default, this bot will look into [wordpress-mobile/test-cases repo](https://github.com/wordpress-mobile/test-cases) for test cases and [file mapping](https://github.com/wordpress-mobile/test-cases/blob/master/config/mapping.json). 10 | 11 | It is possible to override default configuration by adding a `.github/test-case-reminder.json` into a repository where this bot is installed. Config accepts 3 parameters: 12 | 13 | - `tests_repo` - Repository with test suites and `mapping.json` 14 | - `tests_dir` - Path to test-suites directory 15 | - `mapping_file` - Path to `mapping.json` file 16 | - `comment_footer` - Markdown-formatted text that will be added _after_ list of test cases. Useful for suggestions on how to improve/extend existing test cases 17 | 18 | Example config: 19 | 20 | ```json 21 | { 22 | "tests_repo": "brbrr/jetpack", 23 | "tests_dir": "docs/regression-checklist/test-suites/", 24 | "mapping_file": "docs/regression-checklist/mapping.json", 25 | "comment_footer": "Text explaining how to extend and improve existing test suites" 26 | } 27 | ``` 28 | 29 | ## Contribution 30 | 31 | ### Install 32 | 33 | To run the code, make sure you have [Bundler](http://gembundler.com/) installed; then enter `bundle install` on the command line. 34 | 35 | ### Set environment variables 36 | 37 | 1. Create a copy of the `.env-example` file called `.env`. 38 | 2. Create a test app with the following permissions: "Read access to code", "Read access to metadata", "Read and write access to pull requests" 39 | 3. Add your GitHub App's private key, app ID, and webhook secret to the `.env` file. 40 | 41 | ### Run the server 42 | 43 | 1. Run `bundle exec ruby server.rb` on the command line. 44 | 2. View the default Sinatra app at `localhost:3000`. 45 | 46 | ### Develop 47 | 48 | [This guide](https://developer.github.com/apps/quickstart-guides/setting-up-your-development-environment/) will walk through the steps needed to configure a GitHub App and run it on a server. 49 | 50 | After completing the necessary steps in the guide you can use this command in this directory to run the smee client(replacing `https://smee.io/4OcZnobezZzAyaw` with your own domain): 51 | 52 | > smee --url https://smee.io/4OcZnobezZzAyaw --path /event_handler --port 3000 53 | 54 | ### Reload Changes 55 | 56 | If you want server to reload automatically as you save the file you can start the server as below instead of using `ruby server.rb`: 57 | 58 | > bundle exec rerun 'ruby server.rb' 59 | 60 | ### Unit tests 61 | 62 | Unit tests live in `unittest.rb` 63 | 64 | If you want to test potential changes to [mapping.json](https://github.com/wordpress-mobile/test-cases/blob/master/config/mapping.json) file you can first apply the changes to `test_mapping.json` in this repo and test in your local as explained below. 65 | 66 | - Checkout this repo 67 | - Change `test_mapping.json` 68 | - Change `unittests.rb` [this line](https://github.com/wordpress-mobile/test-case-reminder-bot/blob/e12c02305f31bf6c3c6d76f9f3d370c0b4703d3e/unittests.rb#L27) with the filenames you want to test with. 69 | - Change assertions accordingly 70 | 71 | Run below command to run unittests: 72 | 73 | > ruby unittests.rb 74 | -------------------------------------------------------------------------------- /unittests.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | 3 | require 'octokit' 4 | require "base64" 5 | require 'json' 6 | require 'logger' 7 | require 'ostruct' 8 | require 'test/unit' 9 | require "test/unit/rr" 10 | 11 | require './server.rb' 12 | require './application_helper.rb' 13 | 14 | ENV["RAILS_ENV"] = "test" 15 | 16 | class DummyClass 17 | include ApplicationHelper 18 | end 19 | 20 | # Run 'ruby unittests.rb' 21 | class TestAdd < Test::Unit::TestCase 22 | def test_match 23 | 24 | logger = Logger.new(STDOUT) 25 | 26 | ghApp = GHAapp.new() 27 | 28 | content = File.read("test_mapping.json").chomp 29 | json = JSON.parse(content) 30 | logger.debug("json: ") 31 | logger.debug(json) 32 | 33 | # Represents the files changed in a PR 34 | filenames = [ 'gutenberg/packages/block-editor/src/components/media-upload/index.native.js', 35 | 'WordPress/Classes/ViewRelated/Gutenberg/Processors/GutenbergGalleryUploadProcessor.swift', 36 | 'WordPress/Classes/ViewRelated/Gutenberg/Processors/GutenbergImgUploadProcessor.swift', 37 | 'WordPress/Classes/ViewRelated/Gutenberg/Processors/GutenbergVideoUploadProcessor.swift' ] 38 | 39 | files = [] 40 | filenames.each { |filename| puts 41 | newfile = OpenStruct.new 42 | newfile.filename = filename 43 | files << newfile 44 | } 45 | 46 | test_suites = ghApp.helpers.find_matched_testcase_files(files, json, logger) 47 | logger.debug("matched suites: ") 48 | logger.debug(test_suites) 49 | 50 | assert_equal test_suites.length(), 7 51 | assert test_suites.include?("gutenberg/image-upload.md") 52 | assert test_suites.include?("gutenberg/video-upload.md") 53 | assert test_suites.include?("gutenberg/media-text-upload.md") 54 | assert test_suites.include?("gutenberg/gallery-upload-processor.md") 55 | assert test_suites.include?("gutenberg/multi-block-media-upload.md") 56 | assert test_suites.include?("gutenberg/img-upload-processor.md") 57 | assert test_suites.include?("gutenberg/video-upload-processor.md") 58 | end 59 | 60 | def test_default_config 61 | ghApp = GHAapp.new() 62 | default_config = ghApp.helpers.default_config() 63 | expected_config = { 64 | tests_dir: 'test-suites/', 65 | mapping_file: 'config/mapping.json', 66 | tests_repo: 'wordpress-mobile/test-cases', 67 | comment_footer: "\n\nIf you think that suggestions should be improved please edit the configuration file [here](https://github.com/wordpress-mobile/test-cases/blob/master/config/mapping.json). You can also modify/add [test-suites](https://github.com/wordpress-mobile/test-cases/tree/master/test-suites) to be used in the [configuration](https://github.com/wordpress-mobile/test-cases/blob/master/config/mapping.json).\n\n If you are a beginner in mobile platforms follow [build instructions](https://github.com/wordpress-mobile/test-cases/blob/master/README.md#build-instructions).", 68 | } 69 | assert_equal(expected_config, default_config) 70 | end 71 | 72 | def test_fetch_repo_config 73 | ghApp = DummyClass.new() 74 | logger = Logger.new(STDOUT) 75 | 76 | expected_config = { 77 | tests_dir: 'tests_dir', 78 | mapping_file: 'mapping_file', 79 | tests_repo: 'tests_repo', 80 | comment_footer: 'comment_footer' 81 | } 82 | content = Base64.encode64(expected_config.to_json) 83 | 84 | any_instance_of(Octokit::Client) do |klass| 85 | stub(klass).contents { { content: content } } 86 | end 87 | 88 | config = ghApp.fetch_config('test-repo', logger) 89 | assert_equal(expected_config, config) 90 | end 91 | 92 | def test_fetch_default_config 93 | ghApp = DummyClass.new() 94 | logger = Logger.new(STDOUT) 95 | 96 | default_config = ghApp.default_config() 97 | 98 | any_instance_of(Octokit::Client) do |klass| 99 | stub(klass).contents { throw Exception.new() } 100 | end 101 | 102 | config = ghApp.fetch_config('test-repo', logger) 103 | assert_equal(default_config, config) 104 | end 105 | end 106 | 107 | -------------------------------------------------------------------------------- /application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | # Expects that the private key in PEM format. Converts the newlines 3 | PRIVATE_KEY = ( ENV['GITHUB_PRIVATE_KEY'] && !ENV['GITHUB_PRIVATE_KEY'].empty? ) ? OpenSSL::PKey::RSA.new(ENV['GITHUB_PRIVATE_KEY'].gsub('\n', "\n")) : "" 4 | 5 | # Your registered app must have a secret set. The secret is used to verify 6 | # that webhooks are sent by GitHub. 7 | WEBHOOK_SECRET = ENV['GITHUB_WEBHOOK_SECRET'] 8 | 9 | # The GitHub App's identifier (type integer) set when registering an app. 10 | APP_IDENTIFIER = ENV['GITHUB_APP_IDENTIFIER'] 11 | 12 | def handle_pullrequest_opened_event(payload) 13 | repo = payload['repository']['full_name'] 14 | pr_number = payload['pull_request']['number'] 15 | 16 | logger.debug('A PR was opened in repo: ' + repo.to_s + ', PR number: ' + pr_number.to_s) 17 | 18 | config = fetch_config(repo, logger) 19 | mapping_json = fetch_mapping_json( config ) 20 | matched_files = find_matched_testcase_files_from_pr(repo, pr_number, mapping_json) 21 | test_content = create_test_content(matched_files, config) 22 | 23 | if matched_files.count > 0 24 | create_pull_request_review(repo, pr_number, test_content) 25 | end 26 | end 27 | 28 | def fetch_mapping_json(config) 29 | mapping = Octokit.contents(config[:tests_repo], :path => config[:mapping_file]) 30 | content = mapping['content'] 31 | plain = Base64.decode64(content) 32 | json = JSON.parse(plain) 33 | end 34 | 35 | def default_config() 36 | { 37 | tests_dir: 'test-suites/', 38 | mapping_file: 'config/mapping.json', 39 | tests_repo: 'wordpress-mobile/test-cases', 40 | comment_footer: "\n\nIf you think that suggestions should be improved please edit the configuration file [here](https://github.com/wordpress-mobile/test-cases/blob/master/config/mapping.json). You can also modify/add [test-suites](https://github.com/wordpress-mobile/test-cases/tree/master/test-suites) to be used in the [configuration](https://github.com/wordpress-mobile/test-cases/blob/master/config/mapping.json).\n\n If you are a beginner in mobile platforms follow [build instructions](https://github.com/wordpress-mobile/test-cases/blob/master/README.md#build-instructions).", 41 | } 42 | end 43 | 44 | def fetch_config(repo_name, logger) 45 | config_file = '.github/test-case-reminder.json' 46 | config = default_config 47 | 48 | begin 49 | response = Octokit.contents(repo_name, :path => config_file) 50 | rescue => exception 51 | logger.debug('No config file found. Falling back to default values') 52 | return config; 53 | end 54 | 55 | content = response[:content] 56 | plain = Base64.decode64(content) 57 | json = JSON.parse(plain, {symbolize_names: true}) 58 | logger.debug("Found a config file for #{ repo_name }: #{ json }") 59 | 60 | config.each do |key, value| 61 | config[key] = json[key] if ( json[key] ) 62 | end 63 | end 64 | 65 | def create_pull_request_review(repo, pr_number, test_content) 66 | options = { event: 'COMMENT', body: test_content } 67 | @installation_client.create_pull_request_review(repo, pr_number, options) 68 | logger.debug('Created pull request review:') 69 | logger.debug(test_content) 70 | end 71 | 72 | def find_matched_testcase_files_from_pr(repo, pr_number, mapping_json) 73 | files = @installation_client.pull_request_files(repo, pr_number) 74 | matched_files = find_matched_testcase_files(files, mapping_json, logger) 75 | end 76 | 77 | def find_matched_testcase_files(files, mapping_json, logger) 78 | logger.debug('Starting to run regex phrases on file names from the PR.') 79 | matched_files = [] 80 | files.each { |file| puts 81 | filename = file['filename'] 82 | logger.debug('filename: ' + filename.to_s) 83 | mapping_json.each { |item| puts 84 | regex = Regexp.new(item['regex'].to_s) 85 | logger.debug('regex: ') 86 | logger.debug(regex) 87 | if regex.match?(filename) 88 | logger.debug('match! ' + item['testFile'].join(', ')) 89 | matched_files << item['testFile'] 90 | end 91 | } 92 | } 93 | matched_files = matched_files.flatten.uniq 94 | end 95 | 96 | def create_test_content(matched_files, config) 97 | testContent = "Here are some suggested test cases for this PR. \n\n" 98 | matched_files.each { |file| puts 99 | testFile = Octokit.contents(config[:tests_repo], :path => config[:tests_dir] + file) 100 | testContent = testContent + Base64.decode64(testFile['content']) 101 | } 102 | testContent = testContent + config[:comment_footer] 103 | end 104 | 105 | # Saves the raw payload and converts the payload to JSON format 106 | def get_payload_request(request) 107 | # request.body is an IO or StringIO object 108 | # Rewind in case someone already read it 109 | request.body.rewind 110 | # The raw text of the body is required for webhook signature verification 111 | @payload_raw = request.body.read 112 | begin 113 | @payload = JSON.parse @payload_raw 114 | rescue => e 115 | fail "Invalid JSON (#{e}): #{@payload_raw}" 116 | end 117 | end 118 | 119 | # Instantiate an Octokit client authenticated as a GitHub App. 120 | # GitHub App authentication requires that you construct a 121 | # JWT (https://jwt.io/introduction/) signed with the app's private key, 122 | # so GitHub can be sure that it came from the app and wasn't alterered by 123 | # a malicious third party. 124 | def authenticate_app 125 | payload = { 126 | # The time that this JWT was issued, _i.e._ now. 127 | iat: Time.now.to_i, 128 | 129 | # JWT expiration time (10 minute maximum) 130 | exp: Time.now.to_i + (10 * 60), 131 | 132 | # Your GitHub App's identifier number 133 | iss: APP_IDENTIFIER 134 | } 135 | 136 | # Cryptographically sign the JWT. 137 | jwt = JWT.encode(payload, PRIVATE_KEY, 'RS256') 138 | 139 | # Create the Octokit client, using the JWT as the auth token. 140 | @app_client ||= Octokit::Client.new(bearer_token: jwt) 141 | end 142 | 143 | # Instantiate an Octokit client, authenticated as an installation of a 144 | # GitHub App, to run API operations. 145 | def authenticate_installation(payload) 146 | @installation_id = payload['installation']['id'] 147 | @installation_token = @app_client.create_app_installation_access_token(@installation_id)[:token] 148 | @installation_client = Octokit::Client.new(bearer_token: @installation_token) 149 | end 150 | 151 | # Check X-Hub-Signature to confirm that this webhook was generated by 152 | # GitHub, and not a malicious third party. 153 | # 154 | # GitHub uses the WEBHOOK_SECRET, registered to the GitHub App, to 155 | # create the hash signature sent in the `X-HUB-Signature` header of each 156 | # webhook. This code computes the expected hash signature and compares it to 157 | # the signature sent in the `X-HUB-Signature` header. If they don't match, 158 | # this request is an attack, and you should reject it. GitHub uses the HMAC 159 | # hexdigest to compute the signature. The `X-HUB-Signature` looks something 160 | # like this: "sha1=123456". 161 | # See https://developer.github.com/webhooks/securing/ for details. 162 | def verify_webhook_signature 163 | their_signature_header = request.env['HTTP_X_HUB_SIGNATURE'] || 'sha1=' 164 | method, their_digest = their_signature_header.split('=') 165 | our_digest = OpenSSL::HMAC.hexdigest(method, WEBHOOK_SECRET, @payload_raw) 166 | halt 401 unless their_digest == our_digest 167 | 168 | # The X-GITHUB-EVENT header provides the name of the event. 169 | # The action value indicates the which action triggered the event. 170 | logger.debug "---- received event #{request.env['HTTP_X_GITHUB_EVENT']}" 171 | logger.debug "---- action #{@payload['action']}" unless @payload['action'].nil? 172 | end 173 | end --------------------------------------------------------------------------------