├── .gitignore ├── homu ├── git_helper.py ├── html │ ├── build_res.html │ ├── index.html │ └── queue.html ├── utils.py ├── action.py ├── server.py └── main.py ├── .travis.yml ├── setup.py ├── LICENSE ├── cfg.sample.toml ├── README.md └── tests ├── test_main.py └── test_action.py /.gitignore: -------------------------------------------------------------------------------- 1 | /homu/__pycache__/ 2 | /tests/__pycache__/ 3 | /.venv/ 4 | /cfg.toml 5 | /cfg.json 6 | /homu.egg-info/ 7 | /main.db 8 | /cache 9 | **/*.pyc 10 | -------------------------------------------------------------------------------- /homu/git_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import os 5 | 6 | SSH_KEY_FILE = os.path.join(os.path.dirname(__file__), '../cache/key') 7 | 8 | 9 | def main(): 10 | args = ['ssh', '-i', SSH_KEY_FILE, '-S', 'none'] + sys.argv[1:] 11 | os.execvp('ssh', args) 12 | 13 | 14 | if __name__ == '__main__': 15 | main() 16 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - 3.4 5 | - 3.5 6 | - 3.6 7 | install: 8 | - pip install flake8 9 | - pip install 'github3.py<1.0' 10 | - pip install 'toml' 11 | - pip install 'Jinja2' 12 | - pip install 'requests' 13 | - pip install 'bottle' 14 | - pip install 'waitress' 15 | - pip install 'retrying' 16 | script: 17 | - flake8 homu 18 | - python -m unittest discover tests 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='homu', 5 | version='0.3.0', 6 | author='Barosl Lee', 7 | url='https://github.com/barosl/homu', 8 | description=('A bot that integrates with GitHub ' 9 | 'and your favorite continuous integration service'), 10 | 11 | packages=['homu'], 12 | install_requires=[ 13 | 'github3.py==0.9.6', 14 | 'toml', 15 | 'Jinja2', 16 | 'requests', 17 | 'bottle', 18 | 'waitress', 19 | 'retrying', 20 | ], 21 | package_data={ 22 | 'homu': [ 23 | 'html/*.html', 24 | ], 25 | }, 26 | entry_points={ 27 | 'console_scripts': [ 28 | 'homu=homu.main:main', 29 | ], 30 | }, 31 | zip_safe=False, 32 | ) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Barosl Lee 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /homu/html/build_res.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu build result {{repo_label}}#{{pull}} 6 | 31 | 32 | 33 |

Homu build results - {{repo_label}}#{{pull}}

34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | {% for builder in builders %} 47 | 48 | 49 | 50 | 51 | 52 | {% endfor %} 53 | 54 |
Sort keyBuilderStatus
{{loop.index}}{{builder.name}}{{builder.result}}
55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /homu/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import github3 3 | import logging 4 | import subprocess 5 | import sys 6 | import traceback 7 | import requests 8 | import time 9 | 10 | 11 | def github_set_ref(repo, ref, sha, *, force=False, auto_create=True, retry=1): 12 | url = repo._build_url('git', 'refs', ref, base_url=repo._api) 13 | data = {'sha': sha, 'force': force} 14 | 15 | try: 16 | js = repo._json(repo._patch(url, data=json.dumps(data)), 200) 17 | except github3.models.GitHubError as e: 18 | if e.code == 422 and auto_create: 19 | try: 20 | return repo.create_ref('refs/' + ref, sha) 21 | except github3.models.GitHubError: 22 | raise e 23 | elif e.code == 422 and retry > 0: 24 | time.sleep(5) 25 | return github_set_ref(repo, 26 | ref, 27 | sha, 28 | force=force, 29 | auto_create=auto_create, 30 | retry=retry - 1) 31 | else: 32 | raise 33 | 34 | return github3.git.Reference(js, repo) if js else None 35 | 36 | 37 | class Status(github3.repos.status.Status): 38 | def __init__(self, info): 39 | super(Status, self).__init__(info) 40 | 41 | self.context = info.get('context') 42 | 43 | 44 | def github_iter_statuses(repo, sha): 45 | url = repo._build_url('statuses', sha, base_url=repo._api) 46 | return repo._iter(-1, url, Status) 47 | 48 | 49 | def github_create_status(repo, sha, state, target_url='', description='', *, 50 | context=''): 51 | data = {'state': state, 'target_url': target_url, 52 | 'description': description, 'context': context} 53 | url = repo._build_url('statuses', sha, base_url=repo._api) 54 | js = repo._json(repo._post(url, data=data), 201) 55 | return Status(js) if js else None 56 | 57 | 58 | def remove_url_keys_from_json(json): 59 | if isinstance(json, dict): 60 | return {key: remove_url_keys_from_json(value) 61 | for key, value in json.items() 62 | if not key.endswith('url')} 63 | elif isinstance(json, list): 64 | return [remove_url_keys_from_json(value) for value in json] 65 | else: 66 | return json 67 | 68 | 69 | def lazy_debug(logger, f): 70 | if logger.isEnabledFor(logging.DEBUG): 71 | logger.debug(f()) 72 | 73 | 74 | def logged_call(args): 75 | try: 76 | subprocess.check_call(args, stdout=subprocess.DEVNULL, stderr=None) 77 | except subprocess.CalledProcessError: 78 | print('* Failed to execute command: {}'.format(args)) 79 | raise 80 | 81 | 82 | def silent_call(args): 83 | return subprocess.call( 84 | args, 85 | stdout=subprocess.DEVNULL, 86 | stderr=subprocess.DEVNULL, 87 | ) 88 | 89 | 90 | def retry_until(inner, fail, state): 91 | err = None 92 | exc_info = None 93 | 94 | for i in range(3, 0, -1): 95 | try: 96 | inner() 97 | except (github3.models.GitHubError, requests.exceptions.RequestException) as e: # noqa 98 | print('* Intermittent GitHub error: {}'.format(e), file=sys.stderr) 99 | 100 | err = e 101 | exc_info = sys.exc_info() 102 | 103 | if i != 1: 104 | time.sleep(1) 105 | else: 106 | err = None 107 | break 108 | 109 | if err: 110 | print('* GitHub failure in {}'.format(state), file=sys.stderr) 111 | traceback.print_exception(*exc_info) 112 | 113 | fail(err) 114 | -------------------------------------------------------------------------------- /homu/html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu 6 | 18 | 19 | 20 |

Homu

21 | 22 |

Repositories

23 | 24 | 29 | 30 |
31 | 32 |

Homu Cheatsheet

33 | 34 |

Commands

35 | 36 |

37 | Here's a quick reference for the commands Homu accepts. Commands must be posted as 38 | comments on the PR they refer to. Comments may include multiple commands. Homu will 39 | only listen to official reviewers that it is configured to listen to. A comment 40 | must mention the GitHub account Homu is configured to use. (e.g. for the Rust project this is @bors) 41 |

42 | 43 | 58 | 59 |

Examples

60 | 61 | 67 | 68 |

Customizing the Queue's Contents

69 | 70 |

71 | Homu provides a few simple ways to customize the queue's contents to fit your needs: 72 |

73 | 74 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /cfg.sample.toml: -------------------------------------------------------------------------------- 1 | # Priority values above max_priority will be refused. 2 | max_priority = 9001 3 | 4 | [github] 5 | 6 | # Information for securely interacting with GitHub. These are found/generated 7 | # under . 8 | 9 | # A GitHub personal access token. 10 | access_token = "" 11 | 12 | # A GitHub oauth application for this instance of homu: 13 | app_client_id = "" 14 | app_client_secret = "" 15 | 16 | 17 | [git] 18 | 19 | # Use the local Git command. Required to use some advanced features. It also 20 | # speeds up Travis by reducing temporary commits. 21 | #local_git = false 22 | 23 | # SSH private key. Needed only when the local Git command is used. 24 | #ssh_key = """ 25 | #""" 26 | 27 | # By default, Homu extracts the name+email from the Github account it will be 28 | # using. However, you may want to use a private email for the account, and 29 | # associate the commits with a public email address. 30 | #user = "Some Cool Project Bot" 31 | #email = "coolprojectbot-devel@example.com" 32 | 33 | [web] 34 | 35 | # The port homu listens on. 36 | port = 54856 37 | 38 | # Synchronize all open PRs on startup. "Synchronize" means fetch the state of 39 | # all open PRs. 40 | sync_on_start = true 41 | 42 | # Custom hooks can be added as well. 43 | # Homu will ping the given endpoint with POSTdata of the form: 44 | # {'body': 'comment body', 'extra_data': 'extra data', 'pull': pull req number} 45 | # The extra data is the text specified in `@homu hookname=text` 46 | # 47 | # [hooks.hookname] 48 | # trigger = "hookname" # will be triggered by @homu hookname or @homu hookname=text 49 | # endpoint = "http://path/to/endpoint" 50 | # access = "try" # access level required 51 | # has_response = true # Should the response be posted back to github? Only allowed if realtime=true 52 | # realtime = true # Should it only run in realtime mode? If false, this will be replayed each time homu is started (probably not what you want) 53 | 54 | # An example configuration for repository (there can be many of these). NAME 55 | # refers to your repo name. 56 | [repo.NAME] 57 | 58 | # Which repo are we talking about? You can get these fields from your repo URL: 59 | # github.com// 60 | owner = "" 61 | name = "" 62 | 63 | # Who can approve PRs (r+ rights)? You can put GitHub usernames here. 64 | reviewers = [] 65 | # Alternatively, set this allow any github collaborator; 66 | # note that you can *also* specify reviewers above. 67 | # auth_collaborators = true 68 | 69 | # Who has 'try' rights? (try, retry, force, clean, prioritization). It's fine to 70 | # keep this empty. 71 | try_users = [] 72 | 73 | # Keep the commit history linear. Requires the local Git command. 74 | #linear = false 75 | 76 | # Auto-squash commits. Requires the local Git command. 77 | #autosquash = true 78 | 79 | # If the PR already has the same success statuses that we expect on the `auto` 80 | # branch, then push directly to branch if safe to do so. Requires the local Git 81 | # command. 82 | #status_based_exemption = false 83 | 84 | # Maximum test duration allowed for testing a PR in this repository. 85 | # Default to 10 hours. 86 | #timeout = 36000 87 | 88 | # Branch names. These settings are the defaults; it makes sense to leave these 89 | # as-is. 90 | #[repo.NAME.branch] 91 | # 92 | #auto = "auto" 93 | #try = "try" 94 | #rollup = "rollup" 95 | 96 | [repo.NAME.github] 97 | # Arbitrary secret. You can generate one with: openssl rand -hex 20 98 | secret = "" 99 | 100 | # Remove and add GitHub labels when some event happened. 101 | # See servo/homu#141 for detail. 102 | # 103 | #[repo.NAME.labels.approved] # after homu received `r+` 104 | #[repo.NAME.labels.rejected] # after homu received `r-` 105 | #[repo.NAME.labels.conflict] # a merge conflict is detected 106 | #[repo.NAME.labels.succeed] # test successful 107 | #[repo.NAME.labels.failed] # test failed 108 | #[repo.NAME.labels.exempted] # test exempted 109 | #[repo.NAME.labels.timed_out] # test timed out (after 10 hours) 110 | #[repo.NAME.labels.interrupted] # test interrupted (buildbot only) 111 | #[repo.NAME.labels.try] # after homu received `try` 112 | #[repo.NAME.labels.try_succeed] # try-build successful 113 | #[repo.NAME.labels.try_failed] # try-build failed 114 | #[repo.NAME.labels.pushed] # user pushed a commit after `r+`/`try` 115 | #remove = ['list', 'of', 'labels', 'to', 'remove'] 116 | #add = ['list', 'of', 'labels', 'to', 'add'] 117 | #unless = [ 118 | # 'avoid', 'relabeling', 'if', 119 | # 'any', 'of', 'these', 'labels', 'are', 'present', 120 | #] 121 | 122 | # Travis integration. Don't forget to allow Travis to test the `auto` branch! 123 | [repo.NAME.checks.travis] 124 | # Name of the Checks API run. Don't touch this unless you really know what 125 | # you're doing. 126 | name = "Travis CI - Branch" 127 | 128 | # Appveyor integration. Don't forget to allow Appveyor to test the `auto` branch! 129 | #[repo.NAME.status.appveyor] 130 | # 131 | # String label set by status updates. Don't touch this unless you really know 132 | # what you're doing. 133 | #context = 'continuous-integration/appveyor/branch' 134 | 135 | # Generic GitHub Status API support. You don't need this if you're using the 136 | # above examples for Travis/Appveyor. 137 | #[repo.NAME.status.LABEL] 138 | # 139 | # String label set by status updates. 140 | #context = "" 141 | # 142 | # Equivalent context to look for on the PR itself if checking whether the 143 | # build should be exempted. If omitted, looks for the same context. This is 144 | # only used if status_based_exemption is true. 145 | #pr_context = "" 146 | 147 | # Generic GitHub Checks API support. You don't need this if you're using the 148 | # above examples for Travis/Appveyor. 149 | #[repo.NAME.checks.LABEL] 150 | # 151 | # String name of the Checks run. 152 | #name = "" 153 | 154 | # Use buildbot for running tests 155 | #[repo.NAME.buildbot] 156 | # 157 | #url = "" 158 | #secret = "" 159 | # 160 | #builders = ["auto-linux", "auto-mac"] 161 | #try_builders = ["try-linux", "try-mac"] 162 | # 163 | #username = "" 164 | #password = "" 165 | # 166 | # 167 | ## Optional try choosers 168 | ## If adding these, be sure to add associated 169 | ## SingleBranchSchedulers for the try-CHOOSERNAME branch 170 | ## in buildbot's master.cfg 171 | #[repo.NAME.buildbot.try_choosers] 172 | #mac = ["auto-mac-dev", "auto-mac-rel"] 173 | #wpt = ["linux-wpt1", "linux-wpt2"] 174 | # 175 | ## Boolean which indicates whether the builder is included in try builds (defaults to true) 176 | #try = false 177 | 178 | # The database homu uses 179 | [db] 180 | # SQLite file 181 | file = "main.db" 182 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Homu 2 | 3 | [![Hommando]][Hommando Source] 4 | [Who's this?][Akemi Homura] 5 | 6 | Homu is a bot that integrates with GitHub and your favorite continuous 7 | integration service such as [Travis CI], [Appveyor] or [Buildbot]. 8 | 9 | [Hommando]: https://i.imgur.com/j0jNvHF.png 10 | [Hommando Source]: https://www.pixiv.net/artworks/19351345 11 | [Akemi Homura]: https://wiki.puella-magi.net/Homura_Akemi 12 | [Buildbot]: http://buildbot.net/ 13 | [Travis CI]: https://travis-ci.org/ 14 | [Appveyor]: https://www.appveyor.com/ 15 | 16 | ## Why is it needed? 17 | 18 | Let's take Travis CI as an example. If you send a pull request to a repository, 19 | Travis CI instantly shows you the test result, which is great. However, after 20 | several other pull requests are merged into the `master` branch, your pull 21 | request can *still* break things after being merged into `master`. The 22 | traditional continuous integration solutions don't protect you from this. 23 | 24 | In fact, that's why they provide the build status badges. If anything pushed to 25 | `master` is completely free from any breakage, those badges will **not** be 26 | necessary, as they will always be green. The badges themselves prove that there 27 | can still be some breakages, even when continuous integration services are used. 28 | 29 | To solve this problem, the test procedure should be executed *just before the 30 | merge*, not just after the pull request is received. You can manually click the 31 | "restart build" button each time before you merge a pull request, but Homu can 32 | automate this process. It listens to the pull request comments, waiting for an 33 | approval comment from one of the configured reviewers. When the pull request is 34 | approved, Homu tests it using your favorite continuous integration service, and 35 | only when it passes all the tests, it is merged into `master`. 36 | 37 | Note that Homu is **not** a replacement of Travis CI, Buildbot or Appveyor. It 38 | works on top of them. Homu itself doesn't have the ability to test pull 39 | requests. 40 | 41 | ## Influences of bors 42 | 43 | Homu is largely inspired by [bors]. The concept of "tests should be done just 44 | before the merge" came from bors. However, there are also some differences: 45 | 46 | 1. Stateful: Unlike bors, which intends to be stateless, Homu is stateful. It 47 | means that Homu does not need to retrieve all the information again and again 48 | from GitHub at every run. This is essential because of the GitHub's rate 49 | limiting. Once it downloads the initial state, the following changes are 50 | delivered with the [Webhooks] API. 51 | 2. Pushing over polling: Homu prefers pushing wherever possible. The pull 52 | requests from GitHub are retrieved using Webhooks, as stated above. The test 53 | results from Buildbot are pushed back to Homu with the [HttpStatusPush] 54 | feature. This approach improves the overall performance and the response 55 | time, because the bot is informed about the status changes immediately. 56 | 57 | And also, Homu has more features, such as `rollup`, `try`, and the Travis CI & 58 | Appveyor support. 59 | 60 | [bors]: https://github.com/graydon/bors 61 | [Webhooks]: https://developer.github.com/webhooks/ 62 | [HttpStatusPush]: http://docs.buildbot.net/0.8.12/manual/cfg-statustargets.html#httpstatuspush 63 | 64 | ## Usage 65 | 66 | ### How to install 67 | 68 | ```sh 69 | $ sudo apt-get install python3-venv 70 | $ pyvenv .venv 71 | $ . .venv/bin/activate 72 | $ git clone https://github.com/servo/homu.git 73 | $ pip install -e homu 74 | ``` 75 | 76 | ### How to configure 77 | 78 | In the following instructions, `HOST` refers to the hostname (or IP address) 79 | where you are running your custom homu instance. `PORT` is the port the service 80 | is listening to and is configured in `web.port` in `cfg.toml`. `NAME` refers to 81 | the name of the repository you are configuring homu for. 82 | 83 | 1. Copy `cfg.sample.toml` to `cfg.toml`. You'll need to edit this file to set up 84 | your configuration. The following steps explain where you can find important 85 | config values. 86 | 87 | 2. Create a GitHub account that will be used by Homu. You can also use an 88 | existing account. In the [account settings][settings], go to "OAuth 89 | applications" and create a new application: 90 | - Make note of the "Client ID" and "Client Secret"; you will need to put them in 91 | your `cfg.toml`. 92 | - The OAuth Callback URL should be `http://HOST:PORT/callback`. 93 | - The homepage URL isn't necessary; you could set `http://HOST:PORT/`. 94 | 95 | 3. Go to the user settings of the GitHub account you created/used in the 96 | previous step. Go to "Personal access tokens". Click "Generate new token" and 97 | choose the "repo" and "user" scopes. Put the token value in your `cfg.toml`. 98 | 99 | 4. Add your new GitHub account as a Collaborator to the GitHub repo you are 100 | setting up homu for. This can be done in repo (NOT user) "Settings", then 101 | "Collaborators". 102 | 103 | 4.1. Make sure you login as the new GitHub account and that you **accept 104 | the collaborator invitation** you just sent! 105 | 106 | 5. Add a Webhook to your repository. This is done under repo (NOT user) 107 | "Settings", then "Webhooks". Click "Add webhook", the set: 108 | - Payload URL: `http://HOST:PORT/github` 109 | - Content type: `application/json` 110 | - Secret: The same as `repo.NAME.github.secret` in `cfg.toml` 111 | - Events: `Issue Comment`, `Pull Request`, `Pull Request Review Comments`, `Push`, `Status`, `Check runs` 112 | 113 | 6. Add a Webhook to your continuous integration service, if necessary. You don't 114 | need this if using Travis/Appveyor. 115 | - Buildbot 116 | 117 | Insert the following code to the `master.cfg` file: 118 | 119 | ```python 120 | from buildbot.status.status_push import HttpStatusPush 121 | 122 | c['status'].append(HttpStatusPush( 123 | serverUrl='http://HOST:PORT/buildbot', 124 | extra_post_params={'secret': 'repo.NAME.buildbot.secret in cfg.toml'}, 125 | )) 126 | ``` 127 | 128 | 7. Go through the rest of your `cfg.toml` and uncomment (and change, if needed) 129 | parts of the config you'll need. 130 | 131 | [settings]: https://github.com/settings/applications 132 | [travis]: https://travis-ci.org/profile/info 133 | 134 | ### How to run 135 | 136 | ```sh 137 | $ . .venv/bin/activate 138 | $ homu 139 | ``` 140 | 141 | ## Deploying Servo's Homu 142 | 143 | After merging a change to this repo, updated the pinned hash in [Salt]. 144 | 145 | [Salt]: https://github.com/servo/saltfs/blob/master/homu/map.jinja 146 | -------------------------------------------------------------------------------- /homu/html/queue.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu queue - {{repo_label}} {% if treeclosed %} [TREE CLOSED] {% endif %} 6 | 32 | 33 | 34 |

Homu queue - {% if repo_url %}{{repo_label}}{% else %}{{repo_label}}{% endif %} {% if treeclosed %} [TREE CLOSED below priority {{treeclosed}}] {% endif %}

35 | 36 |

37 | 38 | 39 |

40 | 41 |

42 | {{ total }} total, {{ approved }} approved, {{ rolled_up }} rolled up, {{ failed }} failed 43 | / 44 | 45 | / 46 | 47 | 48 |

49 | 50 | 51 | 52 | 53 | 54 | 55 | {% if multiple %} 56 | 57 | {% endif %} 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | {% for state in states %} 71 | 72 | 73 | 74 | {% if multiple %} 75 | 76 | {% endif %} 77 | 78 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | {% endfor %} 93 | 94 |
Sort keyRepository#StatusMergeableTitleHead refAssigneeApproved byPriority
{{loop.index}}{{state.repo_label}}{{state.num}} 79 | {% if state.status == "pending" or state.status == "failure" or state.status == "success" %} 80 | {{state.status}}{{state.status_ext}} 81 | {% else %} 82 | {{state.status}}{{state.status_ext}} 83 | {% endif %} 84 | {{state.mergeable}}{{state.title}}{{state.head_ref}}{{state.assignee}}{{state.approved_by}}{{state.priority}}
95 | 96 | 97 | 98 | 99 | 190 | 191 | 192 | -------------------------------------------------------------------------------- /homu/action.py: -------------------------------------------------------------------------------- 1 | import random 2 | from enum import Enum 3 | 4 | 5 | class LabelEvent(Enum): 6 | APPROVED = 'approved' 7 | REJECTED = 'rejected' 8 | CONFLICT = 'conflict' 9 | SUCCEED = 'succeed' 10 | FAILED = 'failed' 11 | TRY = 'try' 12 | TRY_SUCCEED = 'try_succeed' 13 | TRY_FAILED = 'try_failed' 14 | EXEMPTED = 'exempted' 15 | TIMED_OUT = 'timed_out' 16 | INTERRUPTED = 'interrupted' 17 | PUSHED = 'pushed' 18 | 19 | 20 | PORTAL_TURRET_DIALOG = ["Target acquired", "Activated", "There you are"] 21 | PORTAL_TURRET_IMAGE = "https://cloud.githubusercontent.com/assets/1617736/22222924/c07b2a1c-e16d-11e6-91b3-ac659550585c.png" # noqa 22 | 23 | 24 | def get_portal_turret_dialog(): 25 | return random.choice(PORTAL_TURRET_DIALOG) 26 | 27 | 28 | def still_here(state): 29 | state.add_comment( 30 | ":cake: {}\n\n![]({})".format( 31 | get_portal_turret_dialog(), PORTAL_TURRET_IMAGE) 32 | ) 33 | 34 | 35 | def delegate_to(state, realtime, delegate): 36 | state.delegate = delegate 37 | state.save() 38 | if realtime: 39 | state.add_comment( 40 | ':v: @{} can now approve this pull request' 41 | .format(state.delegate) 42 | ) 43 | 44 | 45 | def set_treeclosed(state, word): 46 | try: 47 | treeclosed = int(word) 48 | state.change_treeclosed(treeclosed) 49 | except ValueError: 50 | pass 51 | state.save() 52 | 53 | 54 | def treeclosed_negative(state): 55 | state.change_treeclosed(-1) 56 | state.save() 57 | 58 | 59 | def hello_or_ping(state): 60 | state.add_comment(":sleepy: I'm awake I'm awake") 61 | 62 | 63 | def rollup(state, word): 64 | state.rollup = word == 'rollup' 65 | state.save() 66 | 67 | 68 | def _try(state, word, realtime, repo_cfg, choose=None): 69 | is_try = word == 'try' 70 | state.try_choose = None 71 | if choose and is_try: 72 | try_choosers = list(repo_cfg.get('try_choosers', [])) 73 | if 'buildbot' in repo_cfg: 74 | try_choosers += list(repo_cfg['buildbot']['try_choosers'].keys()) 75 | if try_choosers: 76 | if choose in try_choosers: 77 | state.try_choose = choose 78 | elif realtime: 79 | state.add_comment( 80 | ':slightly_frowning_face: There is no try chooser {} for this repo, try one of: {}' # noqa 81 | .format(choose, ", ".join(try_choosers)) 82 | ) 83 | return 84 | else: 85 | if realtime: 86 | state.add_comment( 87 | ':slightly_frowning_face: This repo does not have try choosers set up' # noqa 88 | ) 89 | 90 | state.try_ = is_try 91 | state.merge_sha = '' 92 | state.init_build_res([]) 93 | state.save() 94 | if state.try_: 95 | # `try-` just resets the `try` bit and doesn't correspond to 96 | # any meaningful labeling events. 97 | state.change_labels(LabelEvent.TRY) 98 | 99 | 100 | def clean(state): 101 | state.merge_sha = '' 102 | state.init_build_res([]) 103 | state.save() 104 | 105 | 106 | def retry(state): 107 | state.set_status('') 108 | event = LabelEvent.TRY if state.try_ else LabelEvent.APPROVED 109 | state.change_labels(event) 110 | 111 | 112 | def delegate_negative(state): 113 | state.delegate = '' 114 | state.save() 115 | 116 | 117 | def review_rejected(state, realtime): 118 | state.approved_by = '' 119 | state.save() 120 | if realtime: 121 | state.change_labels(LabelEvent.REJECTED) 122 | 123 | 124 | def delegate_positive(state, delegate, realtime): 125 | state.delegate = delegate 126 | state.save() 127 | 128 | if realtime: 129 | state.add_comment( 130 | ':v: @{} can now approve this pull request' 131 | .format(state.delegate) 132 | ) 133 | 134 | 135 | def set_priority(state, realtime, priority, cfg): 136 | try: 137 | pvalue = int(priority) 138 | except ValueError: 139 | return False 140 | 141 | if pvalue > cfg['max_priority']: 142 | if realtime: 143 | state.add_comment( 144 | ':stop_sign: Priority higher than {} is ignored.' 145 | .format(cfg['max_priority']) 146 | ) 147 | return False 148 | state.priority = pvalue 149 | state.save() 150 | return True 151 | 152 | 153 | def review_approved(state, realtime, approver, username, 154 | my_username, sha, states): 155 | # Ignore "r=me" 156 | if approver == 'me': 157 | return False 158 | 159 | # Ignore WIP PRs 160 | if any(map(state.title.startswith, [ 161 | 'WIP', 'TODO', '[WIP]', '[TODO]', 162 | ])): 163 | if realtime: 164 | state.add_comment(':clipboard: Looks like this PR is still in progress, ignoring approval') # noqa 165 | return False 166 | 167 | # Sometimes, GitHub sends the head SHA of a PR as 0000000 168 | # through the webhook. This is called a "null commit", and 169 | # seems to happen when GitHub internally encounters a race 170 | # condition. Last time, it happened when squashing commits 171 | # in a PR. In this case, we just try to retrieve the head 172 | # SHA manually. 173 | if all(x == '0' for x in state.head_sha): 174 | if realtime: 175 | state.add_comment( 176 | ':bangbang: Invalid head SHA found, retrying: `{}`' 177 | .format(state.head_sha) 178 | ) 179 | 180 | state.head_sha = state.get_repo().pull_request(state.num).head.sha # noqa 181 | state.save() 182 | 183 | assert any(x != '0' for x in state.head_sha) 184 | 185 | if state.approved_by and realtime and username != my_username: 186 | for _state in states[state.repo_label].values(): 187 | if _state.status == 'pending': 188 | break 189 | else: 190 | _state = None 191 | 192 | lines = [] 193 | 194 | if state.status in ['failure', 'error']: 195 | lines.append('- This pull request previously failed. You should add more commits to fix the bug, or use `retry` to trigger a build again.') # noqa 196 | 197 | if _state: 198 | if state == _state: 199 | lines.append('- This pull request is currently being tested. If there\'s no response from the continuous integration service, you may use `retry` to trigger a build again.') # noqa 200 | else: 201 | lines.append('- There\'s another pull request that is currently being tested, blocking this pull request: #{}'.format(_state.num)) # noqa 202 | 203 | if lines: 204 | lines.insert(0, '') 205 | lines.insert(0, ':bulb: This pull request was already approved, no need to approve it again.') # noqa 206 | 207 | state.add_comment('\n'.join(lines)) 208 | 209 | if sha_cmp(sha, state.head_sha): 210 | state.approved_by = approver 211 | state.try_ = False 212 | state.try_choose = None 213 | state.set_status('') 214 | 215 | state.save() 216 | elif realtime and username != my_username: 217 | if sha: 218 | msg = '`{}` is not a valid commit SHA.'.format(sha) 219 | state.add_comment( 220 | ':scream_cat: {} Please try again with `{:.7}`.' 221 | .format(msg, state.head_sha) 222 | ) 223 | else: 224 | state.add_comment( 225 | ':pushpin: Commit {:.7} has been approved by `{}`\n\n' # noqa 226 | .format( 227 | state.head_sha, 228 | approver, 229 | my_username, 230 | approver, 231 | state.head_sha, 232 | )) 233 | treeclosed = state.blocked_by_closed_tree() 234 | if treeclosed: 235 | state.add_comment( 236 | ':evergreen_tree: The tree is currently closed for pull requests below priority {}, this pull request will be tested once the tree is reopened' # noqa 237 | .format(treeclosed) 238 | ) 239 | state.change_labels(LabelEvent.APPROVED) 240 | return True 241 | 242 | 243 | def sha_cmp(short, full): 244 | return len(short) >= 4 and short == full[:len(short)] 245 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch, Mock, MagicMock, call 3 | from homu.main import sha_or_blank, force, parse_commands, \ 4 | get_words 5 | 6 | class TestMain(unittest.TestCase): 7 | 8 | def call_parse_commands(self, cfg={}, body='', username='user', repo_cfg={}, 9 | state=None, my_username='my_user', db=None, 10 | states=[], realtime=False, sha=''): 11 | return parse_commands(cfg, body, username, repo_cfg, state, my_username, db, 12 | states, realtime=realtime, sha=sha) 13 | 14 | def test_get_words_no_username(self): 15 | self.assertEqual(get_words("Hi, I'm a test message.", ''), []) 16 | 17 | def test_get_words_incorrect_username(self): 18 | self.assertEqual(get_words("@user I'm a message", 'username'), []) 19 | 20 | def test_get_words_correct_username(self): 21 | self.assertEqual(get_words("@user I'm a message", 'user'), ['@user', "I'm", 'a', 'message']) 22 | 23 | def test_sha_or_blank_return_sha(self): 24 | self.assertEqual(sha_or_blank('f5d42200481'), 'f5d42200481') 25 | 26 | def test_sha_or_blank_return_blank(self): 27 | self.assertEqual(sha_or_blank('f5d@12'), '') 28 | 29 | @patch('homu.main.get_words', return_value=["@bot", "are", "you", "still", "there?"]) 30 | @patch('homu.main.verify_auth', return_value=True) 31 | @patch('homu.main.PullReqState') 32 | @patch('homu.action.still_here') 33 | def test_parse_commands_still_here_realtime(self, mock_still_here, MockPullReqState, mock_auth, mock_words): 34 | state = MockPullReqState() 35 | self.assertFalse(self.call_parse_commands(state=state, realtime=True)) 36 | mock_still_here.assert_called_once_with(state) 37 | 38 | 39 | @patch('homu.main.get_words', return_value=["@bot", "are", "you", "still", "there?"]) 40 | @patch('homu.main.verify_auth', return_value=True) 41 | @patch('homu.main.PullReqState') 42 | @patch('homu.action.still_here') 43 | def test_parse_commands_still_here_not_realtime(self, mock_still_here, MockPullReqState, mock_auth, mock_words): 44 | state = MockPullReqState() 45 | self.assertFalse(self.call_parse_commands(state=state)) 46 | assert not mock_still_here.called, 'still_here was called and should never be.' 47 | 48 | 49 | @patch('homu.main.get_words', return_value=["r+"]) 50 | @patch('homu.main.verify_auth', return_value=True) 51 | @patch('homu.main.PullReqState') 52 | @patch('homu.action.review_approved') 53 | def test_parse_commands_review_approved_verified(self, mock_review_approved, MockPullReqState, mock_auth, mock_words): 54 | state = MockPullReqState() 55 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 56 | mock_review_approved.assert_called_once_with(state, False, 'user', 'user', 'my_user', 'abc123', []) 57 | 58 | @patch('homu.main.get_words', return_value=["r+"]) 59 | @patch('homu.main.verify_auth', return_value=False) 60 | @patch('homu.main.PullReqState') 61 | @patch('homu.action.review_approved') 62 | def test_parse_commands_review_approved_not_verified(self, mock_review_approved, MockPullReqState, mock_auth, mock_words): 63 | state = MockPullReqState() 64 | self.assertFalse(self.call_parse_commands(state=state, sha='abc123')) 65 | assert not mock_review_approved.called, 'mock_review_approved was called and should never be.' 66 | 67 | @patch('homu.main.get_words', return_value=["r=user2"]) 68 | @patch('homu.main.verify_auth', return_value=True) 69 | @patch('homu.main.PullReqState') 70 | @patch('homu.action.review_approved') 71 | def test_parse_commands_review_approved_verified_different_approver(self, mock_review_approved, MockPullReqState, mock_auth, mock_words): 72 | state = MockPullReqState() 73 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 74 | mock_review_approved.assert_called_once_with(state, False, 'user2', 'user', 'my_user', 'abc123', []) 75 | 76 | @patch('homu.main.get_words', return_value=["r-"]) 77 | @patch('homu.main.verify_auth', return_value=True) 78 | @patch('homu.main.PullReqState') 79 | @patch('homu.action.review_rejected') 80 | def test_parse_commands_review_rejected(self, mock_review_rejected, MockPullReqState, mock_auth, mock_words): 81 | state = MockPullReqState() 82 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 83 | mock_review_rejected.assert_called_once_with(state, False) 84 | 85 | @patch('homu.main.get_words', return_value=["p=1"]) 86 | @patch('homu.main.verify_auth', return_value=True) 87 | @patch('homu.main.PullReqState') 88 | @patch('homu.action.set_priority') 89 | def test_parse_commands_set_priority(self, mock_set_priority, MockPullReqState, mock_auth, mock_words): 90 | state = MockPullReqState() 91 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 92 | mock_set_priority.assert_called_once_with(state, False, '1', {}) 93 | 94 | @patch('homu.main.get_words', return_value=["delegate=user2"]) 95 | @patch('homu.main.verify_auth', return_value=True) 96 | @patch('homu.main.PullReqState') 97 | @patch('homu.action.delegate_to') 98 | def test_parse_commands_delegate_to(self, mock_delegate_to, MockPullReqState, mock_auth, mock_words): 99 | state = MockPullReqState() 100 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 101 | mock_delegate_to.assert_called_once_with(state, False, 'user2') 102 | 103 | @patch('homu.main.get_words', return_value=["delegate-"]) 104 | @patch('homu.main.verify_auth', return_value=True) 105 | @patch('homu.main.PullReqState') 106 | @patch('homu.action.delegate_negative') 107 | def test_parse_commands_delegate_negative(self, mock_delegate_negative, MockPullReqState, mock_auth, mock_words): 108 | state = MockPullReqState() 109 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 110 | mock_delegate_negative.assert_called_once_with(state) 111 | 112 | @patch('homu.main.get_words', return_value=["delegate+"]) 113 | @patch('homu.main.verify_auth', return_value=True) 114 | @patch('homu.main.PullReqState') 115 | @patch('homu.action.delegate_positive') 116 | def test_parse_commands_delegate_positive(self, mock_delegate_positive, MockPullReqState, mock_auth, mock_words): 117 | state = MockPullReqState() 118 | state.num = 2 119 | state.get_repo().pull_request(state.num).user.login = 'delegate' 120 | self.assertTrue(self.call_parse_commands(state=state, sha='abc123')) 121 | mock_delegate_positive.assert_called_once_with(state, 'delegate', False) 122 | 123 | @patch('homu.main.get_words', return_value=["retry"]) 124 | @patch('homu.main.verify_auth', return_value=True) 125 | @patch('homu.main.PullReqState') 126 | @patch('homu.action.retry') 127 | def test_parse_commands_retry_realtime(self, mock_retry, MockPullReqState, mock_auth, mock_words): 128 | state = MockPullReqState() 129 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 130 | mock_retry.assert_called_once_with(state) 131 | 132 | @patch('homu.main.get_words', return_value=["retry"]) 133 | @patch('homu.main.verify_auth', return_value=True) 134 | @patch('homu.main.PullReqState') 135 | @patch('homu.action.retry') 136 | def test_parse_commands_retry_not_realtime(self, mock_retry, MockPullReqState, mock_auth, mock_words): 137 | state = MockPullReqState() 138 | self.assertFalse(self.call_parse_commands(state=state, sha='abc123')) 139 | assert not mock_retry.called, 'retry was called and should never be.' 140 | 141 | @patch('homu.main.get_words', return_value=["try"]) 142 | @patch('homu.main.verify_auth', return_value=True) 143 | @patch('homu.main.PullReqState') 144 | @patch('homu.action._try') 145 | def test_parse_commands_try_realtime(self, mock_try, MockPullReqState, mock_auth, mock_words): 146 | state = MockPullReqState() 147 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 148 | mock_try.assert_called_once_with(state, 'try', True, {}) 149 | 150 | @patch('homu.main.get_words', return_value=["try"]) 151 | @patch('homu.main.verify_auth', return_value=True) 152 | @patch('homu.main.PullReqState') 153 | @patch('homu.action._try') 154 | def test_parse_commands_try_not_realtime(self, mock_try, MockPullReqState, mock_auth, mock_words): 155 | state = MockPullReqState() 156 | self.assertFalse(self.call_parse_commands(state=state, sha='abc123')) 157 | assert not mock_try.called, '_try was called and should never be.' 158 | 159 | @patch('homu.main.get_words', return_value=["rollup"]) 160 | @patch('homu.main.verify_auth', return_value=True) 161 | @patch('homu.main.PullReqState') 162 | @patch('homu.action.rollup') 163 | def test_parse_commands_rollup(self, mock_rollup, MockPullReqState, mock_auth, mock_words): 164 | state = MockPullReqState() 165 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 166 | mock_rollup.assert_called_once_with(state, 'rollup') 167 | 168 | @patch('homu.main.get_words', return_value=["clean"]) 169 | @patch('homu.main.verify_auth', return_value=True) 170 | @patch('homu.main.PullReqState') 171 | @patch('homu.action.clean') 172 | def test_parse_commands_clean_realtime(self, mock_clean, MockPullReqState, mock_auth, mock_words): 173 | state = MockPullReqState() 174 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 175 | mock_clean.assert_called_once_with(state) 176 | 177 | @patch('homu.main.get_words', return_value=["clean"]) 178 | @patch('homu.main.verify_auth', return_value=True) 179 | @patch('homu.main.PullReqState') 180 | @patch('homu.action.clean') 181 | def test_parse_commands_clean_not_realtime(self, mock_clean, MockPullReqState, mock_auth, mock_words): 182 | state = MockPullReqState() 183 | self.assertFalse(self.call_parse_commands(state=state, sha='abc123')) 184 | assert not mock_clean.called, 'clean was called and should never be.' 185 | 186 | @patch('homu.main.get_words', return_value=["hello?"]) 187 | @patch('homu.main.verify_auth', return_value=True) 188 | @patch('homu.main.PullReqState') 189 | @patch('homu.action.hello_or_ping') 190 | def test_parse_commands_hello_or_ping_realtime(self, mock_hello_or_ping, MockPullReqState, mock_auth, mock_words): 191 | state = MockPullReqState() 192 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 193 | mock_hello_or_ping.assert_called_once_with(state) 194 | 195 | @patch('homu.main.get_words', return_value=["hello?"]) 196 | @patch('homu.main.verify_auth', return_value=True) 197 | @patch('homu.main.PullReqState') 198 | @patch('homu.action.hello_or_ping') 199 | def test_parse_commands_hello_or_ping_not_realtime(self, mock_hello_or_ping, MockPullReqState, mock_auth, mock_words): 200 | state = MockPullReqState() 201 | self.assertFalse(self.call_parse_commands(state=state, sha='abc123')) 202 | assert not mock_hello_or_ping.called, 'hello_or_ping was called and should never be.' 203 | 204 | @patch('homu.main.get_words', return_value=["treeclosed=1"]) 205 | @patch('homu.main.verify_auth', return_value=True) 206 | @patch('homu.main.PullReqState') 207 | @patch('homu.action.set_treeclosed') 208 | def test_parse_commands_set_treeclosed(self, mock_set_treeclosed, MockPullReqState, mock_auth, mock_words): 209 | state = MockPullReqState() 210 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 211 | mock_set_treeclosed.assert_called_once_with(state, '1') 212 | 213 | @patch('homu.main.get_words', return_value=["treeclosed-"]) 214 | @patch('homu.main.verify_auth', return_value=True) 215 | @patch('homu.main.PullReqState') 216 | @patch('homu.action.treeclosed_negative') 217 | def test_parse_commands_treeclosed_negative(self, mock_treeclosed_negative, MockPullReqState, mock_auth, mock_words): 218 | state = MockPullReqState() 219 | self.assertTrue(self.call_parse_commands(state=state, realtime=True, sha='abc123')) 220 | mock_treeclosed_negative.assert_called_once_with(state) 221 | 222 | 223 | if __name__ == '__main__': 224 | unittest.main() 225 | -------------------------------------------------------------------------------- /tests/test_action.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import unittest 3 | from unittest.mock import patch, call 4 | from homu import action 5 | from homu.action import LabelEvent 6 | 7 | 8 | TRY_CHOOSER_CONFIG = { 9 | "buildbot": { 10 | "builders": ["mac-rel", "mac-wpt", "linux-wpt-1", "linux-wpt-2"], 11 | # keeps the order for testing output 12 | "try_choosers": OrderedDict([ 13 | ("mac", ["mac-rel", "mac-wpt"]), 14 | ("wpt", ["linux-wpt-1", "linux-wpt-2"]) 15 | ]) 16 | }, 17 | "try_choosers": [ 18 | "taskcluster" 19 | ], 20 | } 21 | 22 | TRY_CHOOSER_WITHOUT_BUILDBOT_CONFIG = { 23 | "try_choosers": [ 24 | "taskcluster" 25 | ], 26 | } 27 | 28 | class TestAction(unittest.TestCase): 29 | 30 | @patch('homu.main.PullReqState') 31 | @patch('homu.action.get_portal_turret_dialog', return_value='message') 32 | def test_still_here(self, mock_message, MockPullReqState): 33 | state = MockPullReqState() 34 | action.still_here(state) 35 | state.add_comment.assert_called_once_with(':cake: message\n\n![](https://cloud.githubusercontent.com/assets/1617736/22222924/c07b2a1c-e16d-11e6-91b3-ac659550585c.png)') 36 | 37 | @patch('homu.main.PullReqState') 38 | def test_set_treeclosed(self, MockPullReqState): 39 | state = MockPullReqState() 40 | action.set_treeclosed(state, '123') 41 | state.change_treeclosed.assert_called_once_with(123) 42 | state.save.assert_called_once_with() 43 | 44 | @patch('homu.main.PullReqState') 45 | def test_delegate_to(self, MockPullReqState): 46 | state = MockPullReqState() 47 | action.delegate_to(state, True, 'user') 48 | self.assertEqual(state.delegate, 'user') 49 | state.save.assert_called_once_with() 50 | state.add_comment.assert_called_once_with( 51 | ':v: @user can now approve this pull request' 52 | ) 53 | 54 | @patch('homu.main.PullReqState') 55 | def test_hello_or_ping(self, MockPullReqState): 56 | state = MockPullReqState() 57 | action.hello_or_ping(state) 58 | state.add_comment.assert_called_once_with(":sleepy: I'm awake I'm awake") 59 | 60 | @patch('homu.main.PullReqState') 61 | def test_rollup_positive(self, MockPullReqState): 62 | state = MockPullReqState() 63 | action.rollup(state, 'rollup') 64 | self.assertTrue(state.rollup) 65 | state.save.assert_called_once_with() 66 | 67 | @patch('homu.main.PullReqState') 68 | def test_rollup_negative(self, MockPullReqState): 69 | state = MockPullReqState() 70 | action.rollup(state, 'rollup-') 71 | self.assertFalse(state.rollup) 72 | state.save.assert_called_once_with() 73 | 74 | @patch('homu.main.PullReqState') 75 | def test_try_positive(self, MockPullReqState): 76 | state = MockPullReqState() 77 | action._try(state, 'try', False, {}) 78 | self.assertTrue(state.try_) 79 | state.init_build_res.assert_called_once_with([]) 80 | state.save.assert_called_once_with() 81 | state.change_labels.assert_called_once_with(LabelEvent.TRY) 82 | 83 | @patch('homu.main.PullReqState') 84 | def test_try_negative(self, MockPullReqState): 85 | state = MockPullReqState() 86 | action._try(state, 'try-', False, {}) 87 | self.assertFalse(state.try_) 88 | state.init_build_res.assert_called_once_with([]) 89 | state.save.assert_called_once_with() 90 | assert not state.change_labels.called, 'change_labels was called and should never be.' 91 | 92 | @patch('homu.main.PullReqState') 93 | def test_try_chooser_no_setup(self, MockPullReqState): 94 | state = MockPullReqState() 95 | action._try(state, 'try', True, {}, choose="foo") 96 | self.assertTrue(state.try_) 97 | state.init_build_res.assert_called_once_with([]) 98 | state.save.assert_called_once_with() 99 | state.change_labels.assert_called_once_with(LabelEvent.TRY) 100 | state.add_comment.assert_called_once_with(":slightly_frowning_face: This repo does not have try choosers set up") 101 | 102 | @patch('homu.main.PullReqState') 103 | def test_try_chooser_not_found(self, MockPullReqState): 104 | state = MockPullReqState() 105 | action._try(state, 'try', True, TRY_CHOOSER_CONFIG, choose="foo") 106 | self.assertEqual(state.try_choose, None) 107 | state.init_build_res.assert_not_called() 108 | state.save.assert_not_called() 109 | state.change_labels.assert_not_called() 110 | state.add_comment.assert_called_once_with(":slightly_frowning_face: There is no try chooser foo for this repo, try one of: taskcluster, mac, wpt") 111 | 112 | @patch('homu.main.PullReqState') 113 | def test_try_chooser_found(self, MockPullReqState): 114 | state = MockPullReqState() 115 | action._try(state, 'try', True, TRY_CHOOSER_CONFIG, choose="mac") 116 | self.assertTrue(state.try_) 117 | self.assertEqual(state.try_choose, "mac") 118 | state.init_build_res.assert_called_once_with([]) 119 | state.save.assert_called_once_with() 120 | state.change_labels.assert_called_once_with(LabelEvent.TRY) 121 | 122 | @patch('homu.main.PullReqState') 123 | def test_try_chooser_non_buildbot_found(self, MockPullReqState): 124 | state = MockPullReqState() 125 | action._try(state, 'try', True, TRY_CHOOSER_CONFIG, choose="taskcluster") 126 | self.assertTrue(state.try_) 127 | self.assertEqual(state.try_choose, "taskcluster") 128 | state.init_build_res.assert_called_once_with([]) 129 | state.save.assert_called_once_with() 130 | state.change_labels.assert_called_once_with(LabelEvent.TRY) 131 | 132 | @patch('homu.main.PullReqState') 133 | def test_try_chooser_without_buildbot_found(self, MockPullReqState): 134 | state = MockPullReqState() 135 | action._try(state, 'try', True, TRY_CHOOSER_WITHOUT_BUILDBOT_CONFIG, choose="taskcluster") 136 | self.assertTrue(state.try_) 137 | self.assertEqual(state.try_choose, "taskcluster") 138 | state.init_build_res.assert_called_once_with([]) 139 | state.save.assert_called_once_with() 140 | state.change_labels.assert_called_once_with(LabelEvent.TRY) 141 | 142 | @patch('homu.main.PullReqState') 143 | def test_clean(self, MockPullReqState): 144 | state = MockPullReqState() 145 | action.clean(state) 146 | self.assertEqual(state.merge_sha, '') 147 | state.init_build_res.assert_called_once_with([]) 148 | state.save.assert_called_once_with() 149 | 150 | @patch('homu.main.PullReqState') 151 | def test_retry_try(self, MockPullReqState): 152 | state = MockPullReqState() 153 | state.try_ = True 154 | action.retry(state) 155 | state.set_status.assert_called_once_with('') 156 | state.change_labels.assert_called_once_with(LabelEvent.TRY) 157 | 158 | @patch('homu.main.PullReqState') 159 | def test_treeclosed_negative(self, MockPullReqState): 160 | state = MockPullReqState() 161 | action.treeclosed_negative(state) 162 | state.change_treeclosed.assert_called_once_with(-1) 163 | state.save.assert_called_once_with() 164 | 165 | @patch('homu.main.PullReqState') 166 | def test_retry_approved(self, MockPullReqState): 167 | state = MockPullReqState() 168 | state.try_ = False 169 | action.retry(state) 170 | state.set_status.assert_called_once_with('') 171 | state.change_labels.assert_called_once_with(LabelEvent.APPROVED) 172 | 173 | @patch('homu.main.PullReqState') 174 | def test_delegate_negative(self, MockPullReqState): 175 | state = MockPullReqState() 176 | state.delegate = 'delegate' 177 | action.delegate_negative(state) 178 | self.assertEqual(state.delegate, '') 179 | state.save.assert_called_once_with() 180 | 181 | @patch('homu.main.PullReqState') 182 | def test_delegate_positive_realtime(self, MockPullReqState): 183 | state = MockPullReqState() 184 | action.delegate_positive(state, 'delegate', True) 185 | self.assertEqual(state.delegate, 'delegate') 186 | state.add_comment.assert_called_once_with(':v: @delegate can now approve this pull request') 187 | state.save.assert_called_once_with() 188 | 189 | @patch('homu.main.PullReqState') 190 | def test_delegate_positive_not_realtime(self, MockPullReqState): 191 | state = MockPullReqState() 192 | action.delegate_positive(state, 'delegate', False) 193 | self.assertEqual(state.delegate, 'delegate') 194 | state.save.assert_called_once_with() 195 | assert not state.add_comment.called, 'state.save was called and should never be.' 196 | 197 | @patch('homu.main.PullReqState') 198 | def test_set_priority_not_priority_less_than_max_priority(self, MockPullReqState): 199 | state = MockPullReqState() 200 | action.set_priority(state, True, '1', {'max_priority': 3}) 201 | self.assertEqual(state.priority, 1) 202 | state.save.assert_called_once_with() 203 | 204 | @patch('homu.main.PullReqState') 205 | def test_set_priority_not_priority_more_than_max_priority(self, MockPullReqState): 206 | state = MockPullReqState() 207 | state.priority = 2 208 | self.assertFalse(action.set_priority(state, True, '5', {'max_priority': 3})) 209 | self.assertEqual(state.priority, 2) 210 | state.add_comment.assert_called_once_with(':stop_sign: Priority higher than 3 is ignored.') 211 | assert not state.save.called, 'state.save was called and should never be.' 212 | 213 | @patch('homu.main.PullReqState') 214 | def test_review_approved_approver_me(self, MockPullReqState): 215 | state = MockPullReqState() 216 | self.assertFalse(action.review_approved(state, True, 'me', 'user', 'user', '', [])) 217 | 218 | @patch('homu.main.PullReqState') 219 | def test_review_approved_wip_todo_realtime(self, MockPullReqState): 220 | state = MockPullReqState() 221 | state.title = 'WIP work in progress' 222 | self.assertFalse(action.review_approved(state, True, 'user', 'user', 'user', '', [])) 223 | state.add_comment.assert_called_once_with(':clipboard: Looks like this PR is still in progress, ignoring approval') 224 | 225 | @patch('homu.main.PullReqState') 226 | def test_review_approved_wip_not_realtime(self, MockPullReqState): 227 | state = MockPullReqState() 228 | state.title = 'WIP work in progress' 229 | self.assertFalse(action.review_approved(state, False, 'user', 'user', 'user', '', [])) 230 | assert not state.add_comment.called, 'state.add_comment was called and should never be.' 231 | 232 | @patch('homu.main.PullReqState') 233 | def test_review_approved_equal_usernames(self, MockPullReqState): 234 | state = MockPullReqState() 235 | state.head_sha = 'abcd123' 236 | state.title = "My pull request" 237 | self.assertTrue(action.review_approved(state, True, 'user' ,'user', 'user', 'abcd123', [])) 238 | self.assertEqual(state.approved_by, 'user') 239 | self.assertFalse(state.try_) 240 | state.set_status.assert_called_once_with('') 241 | state.save.assert_called_once_with() 242 | 243 | @patch('homu.main.PullReqState') 244 | def test_review_approved_different_usernames_sha_equals_head_sha(self, MockPullReqState): 245 | state = MockPullReqState() 246 | state.head_sha = 'abcd123' 247 | state.title = "My pull request" 248 | state.repo_label = 'label' 249 | state.status = 'pending' 250 | states = {} 251 | states[state.repo_label] = {'label': state} 252 | self.assertTrue(action.review_approved(state, True, 'user1' ,'user1', 'user2', 'abcd123', states)) 253 | self.assertEqual(state.approved_by, 'user1') 254 | self.assertFalse(state.try_) 255 | state.set_status.assert_called_once_with('') 256 | state.save.assert_called_once_with() 257 | state.add_comment.assert_called_once_with(":bulb: This pull request was already approved, no need to approve it again.\n\n- This pull request is currently being tested. If there's no response from the continuous integration service, you may use `retry` to trigger a build again.") 258 | 259 | @patch('homu.main.PullReqState') 260 | def test_review_approved_different_usernames_sha_different_head_sha(self, MockPullReqState): 261 | state = MockPullReqState() 262 | state.head_sha = 'sdf456' 263 | state.title = "My pull request" 264 | state.repo_label = 'label' 265 | state.status = 'pending' 266 | state.num = 1 267 | states = {} 268 | states[state.repo_label] = {'label': state} 269 | self.assertTrue(action.review_approved(state, True, 'user1', 'user1', 'user2', 'abcd123', states)) 270 | state.add_comment.assert_has_calls([call(":bulb: This pull request was already approved, no need to approve it again.\n\n- This pull request is currently being tested. If there's no response from the continuous integration service, you may use `retry` to trigger a build again."), 271 | call(':scream_cat: `abcd123` is not a valid commit SHA. Please try again with `sdf456`.')]) 272 | 273 | @patch('homu.main.PullReqState') 274 | def test_review_approved_different_usernames_blank_sha_not_blocked_by_closed_tree(self, MockPullReqState): 275 | state = MockPullReqState() 276 | state.blocked_by_closed_tree.return_value = 0 277 | state.head_sha = 'sdf456' 278 | state.title = "My pull request" 279 | state.repo_label = 'label' 280 | state.status = 'pending' 281 | states = {} 282 | states[state.repo_label] = {'label': state} 283 | self.assertTrue(action.review_approved(state, True, 'user1', 'user1', 'user2', '', states)) 284 | state.add_comment.assert_has_calls([call(":bulb: This pull request was already approved, no need to approve it again.\n\n- This pull request is currently being tested. If there's no response from the continuous integration service, you may use `retry` to trigger a build again."), 285 | call(':pushpin: Commit sdf456 has been approved by `user1`\n\n')]) 286 | 287 | @patch('homu.main.PullReqState') 288 | def test_review_approved_different_usernames_blank_sha_blocked_by_closed_tree(self, MockPullReqState): 289 | state = MockPullReqState() 290 | state.blocked_by_closed_tree.return_value = 1 291 | state.head_sha = 'sdf456' 292 | state.title = "My pull request" 293 | state.repo_label = 'label' 294 | state.status = 'pending' 295 | states = {} 296 | states[state.repo_label] = {'label': state} 297 | self.assertTrue(action.review_approved(state, True, 'user1', 'user1', 'user2', '', states)) 298 | state.add_comment.assert_has_calls([call(":bulb: This pull request was already approved, no need to approve it again.\n\n- This pull request is currently being tested. If there's no response from the continuous integration service, you may use `retry` to trigger a build again."), 299 | call(':pushpin: Commit sdf456 has been approved by `user1`\n\n'), 300 | call(':evergreen_tree: The tree is currently closed for pull requests below priority 1, this pull request will be tested once the tree is reopened')]) 301 | state.change_labels.assert_called_once_with(LabelEvent.APPROVED) 302 | 303 | @patch('homu.main.PullReqState') 304 | def test_review_approved_same_usernames_sha_different_head_sha(self, MockPullReqState): 305 | state = MockPullReqState() 306 | state.head_sha = 'sdf456' 307 | state.title = "My pull request" 308 | state.repo_label = 'label' 309 | state.status = 'pending' 310 | states = {} 311 | states[state.repo_label] = {'label': state} 312 | self.assertTrue(action.review_approved(state, True, 'user', 'user', 'user', 'abcd123', states)) 313 | 314 | @patch('homu.main.PullReqState') 315 | def test_review_rejected(self, MockPullReqState): 316 | state = MockPullReqState() 317 | action.review_rejected(state, True) 318 | self.assertEqual(state.approved_by, '') 319 | state.save.assert_called_once_with() 320 | state.change_labels.assert_called_once_with(LabelEvent.REJECTED) 321 | 322 | def test_sha_cmp_equal(self): 323 | self.assertTrue(action.sha_cmp('f259660', 'f259660b128ae59133dff123998ee9b643aff050')) 324 | 325 | def test_sha_cmp_not_equal(self): 326 | self.assertFalse(action.sha_cmp('aaabbb12', 'f259660b128ae59133dff123998ee9b643aff050')) 327 | 328 | def test_sha_cmp_short_length(self): 329 | self.assertFalse(action.sha_cmp('f25', 'f259660b128ae59133dff123998ee9b643aff050')) 330 | 331 | 332 | if __name__ == '__main__': 333 | unittest.main() 334 | -------------------------------------------------------------------------------- /homu/server.py: -------------------------------------------------------------------------------- 1 | import hmac 2 | import json 3 | import urllib.parse 4 | from .main import ( 5 | PullReqState, 6 | parse_commands, 7 | db_query, 8 | INTERRUPTED_BY_HOMU_RE, 9 | synchronize, 10 | ) 11 | from .action import LabelEvent 12 | from . import utils 13 | from .utils import lazy_debug 14 | import github3 15 | import jinja2 16 | import requests 17 | import pkg_resources 18 | from bottle import ( 19 | get, 20 | post, 21 | run, 22 | request, 23 | redirect, 24 | abort, 25 | response, 26 | ) 27 | from threading import Thread 28 | import sys 29 | import os 30 | import traceback 31 | from retrying import retry 32 | 33 | import bottle 34 | bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 10 35 | 36 | 37 | class G: 38 | pass 39 | 40 | 41 | g = G() 42 | 43 | 44 | def find_state(sha): 45 | for repo_label, repo_states in g.states.items(): 46 | for state in repo_states.values(): 47 | if state.merge_sha == sha: 48 | return state, repo_label 49 | 50 | raise ValueError('Invalid SHA') 51 | 52 | 53 | def get_repo(repo_label, repo_cfg): 54 | repo = g.repos[repo_label].gh 55 | if not repo: 56 | repo = g.gh.repository(repo_cfg['owner'], repo_cfg['name']) 57 | g.repos[repo_label] = repo 58 | assert repo.owner.login == repo_cfg['owner'] 59 | assert repo.name == repo_cfg['name'] 60 | return repo 61 | 62 | 63 | @get('/') 64 | def index(): 65 | return g.tpls['index'].render(repos=[g.repos[label] 66 | for label in sorted(g.repos)]) 67 | 68 | 69 | @get('/results//') 70 | def result(repo_label, pull): 71 | if repo_label not in g.states: 72 | abort(404, 'No such repository: {}'.format(repo_label)) 73 | states = [state for state in g.states[repo_label].values() 74 | if state.num == pull] 75 | if len(states) == 0: 76 | return 'No build results for pull request {}'.format(pull) 77 | 78 | state = states[0] 79 | builders = [] 80 | repo_url = 'https://github.com/{}/{}'.format( 81 | g.cfg['repo'][repo_label]['owner'], 82 | g.cfg['repo'][repo_label]['name']) 83 | for (builder, data) in state.build_res.items(): 84 | result = "pending" 85 | if data['res'] is not None: 86 | result = "success" if data['res'] else "failed" 87 | 88 | if not data['url']: 89 | # This happens to old try builds 90 | return 'No build results for pull request {}'.format(pull) 91 | 92 | builders.append({ 93 | 'url': data['url'], 94 | 'result': result, 95 | 'name': builder, 96 | }) 97 | 98 | return g.tpls['build_res'].render(repo_label=repo_label, repo_url=repo_url, 99 | builders=builders, pull=pull) 100 | 101 | 102 | @get('/queue/') 103 | def queue(repo_label): 104 | logger = g.logger.getChild('queue') 105 | 106 | lazy_debug(logger, lambda: 'repo_label: {}'.format(repo_label)) 107 | 108 | single_repo_closed = None 109 | if repo_label == 'all': 110 | labels = g.repos.keys() 111 | multiple = True 112 | repo_url = None 113 | else: 114 | labels = repo_label.split('+') 115 | multiple = len(labels) > 1 116 | if repo_label in g.repos and g.repos[repo_label].treeclosed >= 0: 117 | single_repo_closed = g.repos[repo_label].treeclosed 118 | repo_url = 'https://github.com/{}/{}'.format( 119 | g.cfg['repo'][repo_label]['owner'], 120 | g.cfg['repo'][repo_label]['name']) 121 | 122 | states = [] 123 | for label in labels: 124 | try: 125 | states += g.states[label].values() 126 | except KeyError: 127 | abort(404, 'No such repository: {}'.format(label)) 128 | 129 | pull_states = sorted(states) 130 | rows = [] 131 | for state in pull_states: 132 | treeclosed = (single_repo_closed or 133 | state.priority < g.repos[state.repo_label].treeclosed) 134 | status_ext = '' 135 | 136 | if state.try_: 137 | status_ext += ' (try)' 138 | 139 | if treeclosed: 140 | status_ext += ' [TREE CLOSED]' 141 | 142 | rows.append({ 143 | 'status': state.get_status(), 144 | 'status_ext': status_ext, 145 | 'priority': 'rollup' if state.rollup else state.priority, 146 | 'url': 'https://github.com/{}/{}/pull/{}'.format(state.owner, 147 | state.name, 148 | state.num), 149 | 'num': state.num, 150 | 'approved_by': state.approved_by, 151 | 'title': state.title, 152 | 'head_ref': state.head_ref, 153 | 'mergeable': ('yes' if state.mergeable is True else 154 | 'no' if state.mergeable is False else ''), 155 | 'assignee': state.assignee, 156 | 'repo_label': state.repo_label, 157 | 'repo_url': 'https://github.com/{}/{}'.format(state.owner, 158 | state.name), 159 | 'greyed': "treeclosed" if treeclosed else "", 160 | }) 161 | 162 | return g.tpls['queue'].render( 163 | repo_url=repo_url, 164 | repo_label=repo_label, 165 | treeclosed=single_repo_closed, 166 | states=rows, 167 | oauth_client_id=g.cfg['github']['app_client_id'], 168 | total=len(pull_states), 169 | approved=len([x for x in pull_states if x.approved_by]), 170 | rolled_up=len([x for x in pull_states if x.rollup]), 171 | failed=len([x for x in pull_states if x.status == 'failure' or 172 | x.status == 'error']), 173 | multiple=multiple, 174 | ) 175 | 176 | 177 | @get('/callback') 178 | def callback(): 179 | logger = g.logger.getChild('callback') 180 | 181 | response.content_type = 'text/plain' 182 | 183 | code = request.query.code 184 | state = json.loads(request.query.state) 185 | 186 | lazy_debug(logger, lambda: 'state: {}'.format(state)) 187 | oauth_url = 'https://github.com/login/oauth/access_token' 188 | 189 | try: 190 | res = requests.post(oauth_url, data={ 191 | 'client_id': g.cfg['github']['app_client_id'], 192 | 'client_secret': g.cfg['github']['app_client_secret'], 193 | 'code': code, 194 | }) 195 | except Exception as ex: # noqa 196 | logger.warn('/callback encountered an error ' 197 | 'during github oauth callback') 198 | # probably related to https://gitlab.com/pycqa/flake8/issues/42 199 | lazy_debug( 200 | logger, 201 | lambda ex=ex: 'github oauth callback err: {}'.format(ex), 202 | ) 203 | abort(502, 'Bad Gateway') 204 | 205 | args = urllib.parse.parse_qs(res.text) 206 | token = args['access_token'][0] 207 | 208 | repo_label = state['repo_label'] 209 | repo_cfg = g.repo_cfgs[repo_label] 210 | repo = get_repo(repo_label, repo_cfg) 211 | 212 | user_gh = github3.login(token=token) 213 | 214 | if state['cmd'] == 'rollup': 215 | return rollup(user_gh, state, repo_label, repo_cfg, repo) 216 | elif state['cmd'] == 'synch': 217 | return synch(user_gh, state, repo_label, repo_cfg, repo) 218 | else: 219 | abort(400, 'Invalid command') 220 | 221 | 222 | def rollup(user_gh, state, repo_label, repo_cfg, repo): 223 | user_repo = user_gh.repository(user_gh.user().login, repo.name) 224 | base_repo = user_gh.repository(repo.owner.login, repo.name) 225 | 226 | nums = state.get('nums', []) 227 | if nums: 228 | try: 229 | rollup_states = [g.states[repo_label][num] for num in nums] 230 | except KeyError as e: 231 | return 'Invalid PR number: {}'.format(e.args[0]) 232 | else: 233 | rollup_states = [x for x in g.states[repo_label].values() if x.rollup] 234 | rollup_states = [x for x in rollup_states if x.approved_by] 235 | rollup_states.sort(key=lambda x: x.num) 236 | 237 | if not rollup_states: 238 | return 'No pull requests are marked as rollup' 239 | 240 | base_ref = rollup_states[0].base_ref 241 | 242 | base_sha = repo.ref('heads/' + base_ref).object.sha 243 | utils.github_set_ref( 244 | user_repo, 245 | 'heads/' + repo_cfg.get('branch', {}).get('rollup', 'rollup'), 246 | base_sha, 247 | force=True, 248 | ) 249 | 250 | successes = [] 251 | failures = [] 252 | 253 | for state in rollup_states: 254 | if base_ref != state.base_ref: 255 | failures.append(state.num) 256 | continue 257 | 258 | merge_msg = 'Rollup merge of #{} - {}, r={}\n\n{}\n\n{}'.format( 259 | state.num, 260 | state.head_ref, 261 | state.approved_by, 262 | state.title, 263 | state.body, 264 | ) 265 | 266 | try: 267 | rollup = repo_cfg.get('branch', {}).get('rollup', 'rollup') 268 | user_repo.merge(rollup, state.head_sha, merge_msg) 269 | except github3.models.GitHubError as e: 270 | if e.code != 409: 271 | raise 272 | 273 | failures.append(state.num) 274 | else: 275 | successes.append(state.num) 276 | 277 | title = 'Rollup of {} pull requests'.format(len(successes)) 278 | body = '- Successful merges: {}\n- Failed merges: {}'.format( 279 | ', '.join('#{}'.format(x) for x in successes), 280 | ', '.join('#{}'.format(x) for x in failures), 281 | ) 282 | 283 | try: 284 | rollup = repo_cfg.get('branch', {}).get('rollup', 'rollup') 285 | pull = base_repo.create_pull( 286 | title, 287 | state.base_ref, 288 | user_repo.owner.login + ':' + rollup, 289 | body, 290 | ) 291 | except github3.models.GitHubError as e: 292 | return e.response.text 293 | else: 294 | redirect(pull.html_url) 295 | 296 | 297 | @post('/github') 298 | def github(): 299 | logger = g.logger.getChild('github') 300 | 301 | response.content_type = 'text/plain' 302 | 303 | payload = request.body.read() 304 | info = request.json 305 | 306 | lazy_debug(logger, lambda: 'info: {}'.format(utils.remove_url_keys_from_json(info))) # noqa 307 | 308 | owner_info = info['repository']['owner'] 309 | owner = owner_info.get('login') or owner_info['name'] 310 | repo_label = g.repo_labels[owner, info['repository']['name']] 311 | repo_cfg = g.repo_cfgs[repo_label] 312 | 313 | hmac_method, hmac_sig = request.headers['X-Hub-Signature'].split('=') 314 | if hmac_sig != hmac.new( 315 | repo_cfg['github']['secret'].encode('utf-8'), 316 | payload, 317 | hmac_method, 318 | ).hexdigest(): 319 | abort(400, 'Invalid signature') 320 | 321 | event_type = request.headers['X-Github-Event'] 322 | 323 | if event_type == 'pull_request_review_comment': 324 | action = info['action'] 325 | original_commit_id = info['comment']['original_commit_id'] 326 | head_sha = info['pull_request']['head']['sha'] 327 | 328 | if action == 'created' and original_commit_id == head_sha: 329 | pull_num = info['pull_request']['number'] 330 | body = info['comment']['body'] 331 | username = info['sender']['login'] 332 | 333 | state = g.states[repo_label].get(pull_num) 334 | if state: 335 | state.title = info['pull_request']['title'] 336 | state.body = info['pull_request']['body'] 337 | 338 | if parse_commands( 339 | g.cfg, 340 | body, 341 | username, 342 | repo_cfg, 343 | state, 344 | g.my_username, 345 | g.db, 346 | g.states, 347 | realtime=True, 348 | sha=original_commit_id, 349 | ): 350 | state.save() 351 | 352 | g.queue_handler() 353 | elif event_type == 'pull_request_review': 354 | action = info['action'] 355 | commit_id = info['review']['commit_id'] 356 | head_sha = info['pull_request']['head']['sha'] 357 | 358 | if action == 'submitted' and commit_id == head_sha: 359 | pull_num = info['pull_request']['number'] 360 | body = info['review']['body'] 361 | username = info['sender']['login'] 362 | 363 | state = g.states[repo_label].get(pull_num) 364 | if state: 365 | state.title = info['pull_request']['title'] 366 | state.body = info['pull_request']['body'] 367 | 368 | if parse_commands( 369 | g.cfg, 370 | body, 371 | username, 372 | repo_cfg, 373 | state, 374 | g.my_username, 375 | g.db, 376 | g.states, 377 | realtime=True, 378 | sha=commit_id, 379 | ): 380 | state.save() 381 | 382 | g.queue_handler() 383 | elif event_type == 'pull_request': 384 | action = info['action'] 385 | pull_num = info['number'] 386 | head_sha = info['pull_request']['head']['sha'] 387 | 388 | if action == 'synchronize': 389 | state = g.states[repo_label][pull_num] 390 | state.head_advanced(head_sha) 391 | 392 | state.save() 393 | 394 | elif action in ['opened', 'reopened']: 395 | state = PullReqState(pull_num, head_sha, '', g.db, repo_label, 396 | g.mergeable_que, g.gh, 397 | info['repository']['owner']['login'], 398 | info['repository']['name'], 399 | repo_cfg.get('labels', {}), 400 | g.repos) 401 | state.title = info['pull_request']['title'] 402 | state.body = info['pull_request']['body'] 403 | state.head_ref = info['pull_request']['head']['repo']['owner']['login'] + ':' + info['pull_request']['head']['ref'] # noqa 404 | state.base_ref = info['pull_request']['base']['ref'] 405 | state.set_mergeable(info['pull_request']['mergeable']) 406 | state.assignee = (info['pull_request']['assignee']['login'] if 407 | info['pull_request']['assignee'] else '') 408 | 409 | found = False 410 | 411 | if action == 'reopened': 412 | # FIXME: Review comments are ignored here 413 | for c in state.get_repo().issue(pull_num).iter_comments(): 414 | found = parse_commands( 415 | g.cfg, 416 | c.body, 417 | c.user.login, 418 | repo_cfg, 419 | state, 420 | g.my_username, 421 | g.db, 422 | g.states, 423 | ) or found 424 | 425 | status = '' 426 | for info in utils.github_iter_statuses(state.get_repo(), 427 | state.head_sha): 428 | if info.context == 'homu': 429 | status = info.state 430 | break 431 | 432 | state.set_status(status) 433 | 434 | state.save() 435 | 436 | g.states[repo_label][pull_num] = state 437 | 438 | if found: 439 | g.queue_handler() 440 | 441 | elif action == 'closed': 442 | state = g.states[repo_label][pull_num] 443 | if hasattr(state, 'fake_merge_sha'): 444 | def inner(): 445 | utils.github_set_ref( 446 | state.get_repo(), 447 | 'heads/' + state.base_ref, 448 | state.merge_sha, 449 | force=True, 450 | ) 451 | 452 | def fail(err): 453 | state.add_comment(':boom: Failed to recover from the ' 454 | 'artificial commit. See {} for details.' 455 | ' ({})'.format(state.fake_merge_sha, 456 | err)) 457 | 458 | utils.retry_until(inner, fail, state) 459 | 460 | del g.states[repo_label][pull_num] 461 | 462 | db_query(g.db, 'DELETE FROM pull WHERE repo = ? AND num = ?', 463 | [repo_label, pull_num]) 464 | db_query(g.db, 'DELETE FROM build_res WHERE repo = ? AND num = ?', 465 | [repo_label, pull_num]) 466 | db_query(g.db, 'DELETE FROM mergeable WHERE repo = ? AND num = ?', 467 | [repo_label, pull_num]) 468 | 469 | g.queue_handler() 470 | 471 | elif action in ['assigned', 'unassigned']: 472 | state = g.states[repo_label][pull_num] 473 | state.assignee = (info['pull_request']['assignee']['login'] if 474 | info['pull_request']['assignee'] else '') 475 | 476 | state.save() 477 | 478 | else: 479 | lazy_debug(logger, lambda: 'Invalid pull_request action: {}'.format(action)) # noqa 480 | 481 | elif event_type == 'push': 482 | ref = info['ref'][len('refs/heads/'):] 483 | 484 | for state in list(g.states[repo_label].values()): 485 | if state.base_ref == ref: 486 | state.set_mergeable(None, cause={ 487 | 'sha': info['head_commit']['id'], 488 | 'title': info['head_commit']['message'].splitlines()[0], 489 | }) 490 | 491 | if state.head_sha == info['before']: 492 | if state.status: 493 | state.change_labels(LabelEvent.PUSHED) 494 | state.head_advanced(info['after']) 495 | 496 | state.save() 497 | 498 | elif event_type == 'issue_comment': 499 | body = info['comment']['body'] 500 | username = info['comment']['user']['login'] 501 | pull_num = info['issue']['number'] 502 | 503 | state = g.states[repo_label].get(pull_num) 504 | 505 | if 'pull_request' in info['issue'] and state: 506 | state.title = info['issue']['title'] 507 | state.body = info['issue']['body'] 508 | 509 | if parse_commands( 510 | g.cfg, 511 | body, 512 | username, 513 | repo_cfg, 514 | state, 515 | g.my_username, 516 | g.db, 517 | g.states, 518 | realtime=True, 519 | ): 520 | state.save() 521 | 522 | g.queue_handler() 523 | 524 | elif event_type == 'status': 525 | try: 526 | state, repo_label = find_state(info['sha']) 527 | except ValueError: 528 | return 'OK' 529 | 530 | status_name = "" 531 | if 'status' in repo_cfg: 532 | for name, value in repo_cfg['status'].items(): 533 | if 'context' in value and value['context'] == info['context']: 534 | status_name = name 535 | if status_name == "": 536 | return 'OK' 537 | 538 | if info['state'] == 'pending': 539 | return 'OK' 540 | 541 | for row in info['branches']: 542 | if row['name'] == state.base_ref: 543 | return 'OK' 544 | 545 | report_build_res(info['state'] == 'success', info['target_url'], 546 | 'status-' + status_name, state, logger, repo_cfg) 547 | 548 | elif event_type == 'check_run': 549 | try: 550 | state, repo_label = find_state(info['check_run']['head_sha']) 551 | except ValueError: 552 | return 'OK' 553 | 554 | current_run_name = info['check_run']['name'] 555 | checks_name = None 556 | if 'checks' in repo_cfg: 557 | for name, value in repo_cfg['checks'].items(): 558 | if 'name' in value and value['name'] == current_run_name: 559 | checks_name = name 560 | if checks_name is None: 561 | return 'OK' 562 | 563 | if info['check_run']['status'] != 'completed': 564 | return 'OK' 565 | if info['check_run']['conclusion'] is None: 566 | return 'OK' 567 | 568 | report_build_res( 569 | info['check_run']['conclusion'] == 'success', 570 | info['check_run']['details_url'], 571 | 'checks-' + checks_name, 572 | state, logger, repo_cfg, 573 | ) 574 | 575 | return 'OK' 576 | 577 | 578 | def report_build_res(succ, url, builder, state, logger, repo_cfg): 579 | lazy_debug(logger, 580 | lambda: 'build result {}: builder = {}, succ = {}, current build_res = {}' # noqa 581 | .format(state, builder, succ, 582 | state.build_res_summary())) 583 | 584 | state.set_build_res(builder, succ, url) 585 | 586 | if succ: 587 | if all(x['res'] for x in state.build_res.values()): 588 | state.set_status('success') 589 | desc = 'Test successful' 590 | utils.github_create_status(state.get_repo(), state.head_sha, 591 | 'success', url, desc, context='homu') 592 | 593 | urls = ', '.join('[{}]({})'.format(builder, x['url']) for builder, x in sorted(state.build_res.items())) # noqa 594 | test_comment = ':sunny: {} - {}'.format(desc, urls) 595 | 596 | if state.approved_by and not state.try_: 597 | comment = (test_comment + '\n' + 598 | 'Approved by: {}\nPushing {} to {}...' 599 | ).format(state.approved_by, state.merge_sha, 600 | state.base_ref) 601 | state.add_comment(comment) 602 | state.change_labels(LabelEvent.SUCCEED) 603 | try: 604 | try: 605 | utils.github_set_ref(state.get_repo(), 'heads/' + 606 | state.base_ref, state.merge_sha) 607 | except github3.models.GitHubError: 608 | utils.github_create_status( 609 | state.get_repo(), 610 | state.merge_sha, 611 | 'success', '', 612 | 'Branch protection bypassed', 613 | context='homu') 614 | utils.github_set_ref(state.get_repo(), 'heads/' + 615 | state.base_ref, state.merge_sha) 616 | 617 | state.fake_merge(repo_cfg) 618 | 619 | except github3.models.GitHubError as e: 620 | state.set_status('error') 621 | desc = ('Test was successful, but fast-forwarding failed:' 622 | ' {}'.format(e)) 623 | utils.github_create_status(state.get_repo(), 624 | state.head_sha, 'error', url, 625 | desc, context='homu') 626 | 627 | state.add_comment(':eyes: ' + desc) 628 | else: 629 | comment = (test_comment + '\n' + 630 | 'State: approved={} try={}' 631 | ).format(state.approved_by, state.try_) 632 | state.add_comment(comment) 633 | state.change_labels(LabelEvent.TRY_SUCCEED) 634 | 635 | else: 636 | if state.status == 'pending': 637 | state.set_status('failure') 638 | desc = 'Test failed' 639 | utils.github_create_status(state.get_repo(), state.head_sha, 640 | 'failure', url, desc, context='homu') 641 | 642 | state.add_comment(':broken_heart: {} - [{}]({})'.format(desc, 643 | builder, 644 | url)) 645 | event = LabelEvent.TRY_FAILED if state.try_ else LabelEvent.FAILED 646 | state.change_labels(event) 647 | 648 | g.queue_handler() 649 | 650 | 651 | @post('/buildbot') 652 | def buildbot(): 653 | logger = g.logger.getChild('buildbot') 654 | 655 | response.content_type = 'text/plain' 656 | 657 | for row in json.loads(request.forms.packets): 658 | if row['event'] == 'buildFinished': 659 | info = row['payload']['build'] 660 | lazy_debug(logger, lambda: 'info: {}'.format(info)) 661 | props = dict(x[:2] for x in info['properties']) 662 | 663 | if 'retry' in info['text']: 664 | continue 665 | 666 | if not props['revision']: 667 | continue 668 | 669 | try: 670 | state, repo_label = find_state(props['revision']) 671 | except ValueError: 672 | lazy_debug(logger, 673 | lambda: 'Invalid commit ID from Buildbot: {}'.format(props['revision'])) # noqa 674 | continue 675 | 676 | lazy_debug(logger, lambda: 'state: {}, {}'.format(state, state.build_res_summary())) # noqa 677 | 678 | if info['builderName'] not in state.build_res: 679 | lazy_debug(logger, 680 | lambda: 'Invalid builder from Buildbot: {}'.format(info['builderName'])) # noqa 681 | continue 682 | 683 | repo_cfg = g.repo_cfgs[repo_label] 684 | 685 | if request.forms.secret != repo_cfg['buildbot']['secret']: 686 | abort(400, 'Invalid secret') 687 | 688 | build_succ = 'successful' in info['text'] or info['results'] == 0 689 | 690 | url = '{}/builders/{}/builds/{}'.format( 691 | repo_cfg['buildbot']['url'], 692 | info['builderName'], 693 | props['buildnumber'], 694 | ) 695 | 696 | if 'interrupted' in info['text']: 697 | step_name = '' 698 | for step in reversed(info['steps']): 699 | if 'interrupted' in step.get('text', []): 700 | step_name = step['name'] 701 | break 702 | 703 | if step_name: 704 | try: 705 | url = ('{}/builders/{}/builds/{}/steps/{}/logs/interrupt' # noqa 706 | ).format(repo_cfg['buildbot']['url'], 707 | info['builderName'], 708 | props['buildnumber'], 709 | step_name,) 710 | res = requests.get(url) 711 | except Exception as ex: # noqa 712 | logger.warn('/buildbot encountered an error during ' 713 | 'github logs request') 714 | lazy_debug( 715 | logger, 716 | lambda ex=ex: 'buildbot logs err: {}'.format(ex), 717 | ) 718 | abort(502, 'Bad Gateway') 719 | 720 | mat = INTERRUPTED_BY_HOMU_RE.search(res.text) 721 | if mat: 722 | interrupt_token = mat.group(1) 723 | if getattr(state, 'interrupt_token', 724 | '') != interrupt_token: 725 | state.interrupt_token = interrupt_token 726 | 727 | if state.status == 'pending': 728 | state.set_status('') 729 | 730 | desc = (':snowman: The build was interrupted ' 731 | 'to prioritize another pull request.') 732 | state.add_comment(desc) 733 | state.change_labels(LabelEvent.INTERRUPTED) 734 | utils.github_create_status(state.get_repo(), 735 | state.head_sha, 736 | 'error', url, 737 | desc, 738 | context='homu') 739 | 740 | g.queue_handler() 741 | 742 | continue 743 | 744 | else: 745 | logger.error('Corrupt payload from Buildbot') 746 | 747 | report_build_res(build_succ, url, info['builderName'], 748 | state, logger, repo_cfg) 749 | 750 | elif row['event'] == 'buildStarted': 751 | info = row['payload']['build'] 752 | lazy_debug(logger, lambda: 'info: {}'.format(info)) 753 | props = dict(x[:2] for x in info['properties']) 754 | 755 | if not props['revision']: 756 | continue 757 | 758 | try: 759 | state, repo_label = find_state(props['revision']) 760 | except ValueError: 761 | pass 762 | else: 763 | if info['builderName'] in state.build_res: 764 | repo_cfg = g.repo_cfgs[repo_label] 765 | 766 | if request.forms.secret != repo_cfg['buildbot']['secret']: 767 | abort(400, 'Invalid secret') 768 | 769 | url = '{}/builders/{}/builds/{}'.format( 770 | repo_cfg['buildbot']['url'], 771 | info['builderName'], 772 | props['buildnumber'], 773 | ) 774 | 775 | state.set_build_res(info['builderName'], None, url) 776 | 777 | if g.buildbot_slots[0] == props['revision']: 778 | g.buildbot_slots[0] = '' 779 | 780 | g.queue_handler() 781 | 782 | return 'OK' 783 | 784 | 785 | def synch(user_gh, state, repo_label, repo_cfg, repo): 786 | if not repo.is_collaborator(user_gh.user().login): 787 | abort(400, 'You are not a collaborator') 788 | 789 | Thread(target=synchronize, args=[repo_label, g.cfg, repo_cfg, g.logger, 790 | g.gh, g.states, g.repos, g.db, 791 | g.mergeable_que, g.my_username, 792 | g.repo_labels]).start() 793 | 794 | return 'Synchronizing {}...'.format(repo_label) 795 | 796 | 797 | def synch_all(): 798 | @retry(wait_exponential_multiplier=1000, wait_exponential_max=600000) 799 | def sync_repo(repo_label, g): 800 | try: 801 | synchronize(repo_label, g.cfg, g.repo_cfgs[repo_label], g.logger, 802 | g.gh, g.states, g.repos, g.db, g.mergeable_que, 803 | g.my_username, g.repo_labels) 804 | except Exception: 805 | print('* Error while synchronizing {}'.format(repo_label)) 806 | traceback.print_exc() 807 | raise 808 | 809 | for repo_label in g.repos: 810 | sync_repo(repo_label, g) 811 | print('* Done synchronizing all') 812 | 813 | 814 | @post('/admin') 815 | def admin(): 816 | if request.json['secret'] != g.cfg['web']['secret']: 817 | return 'Authentication failure' 818 | 819 | if request.json['cmd'] == 'repo_new': 820 | repo_label = request.json['repo_label'] 821 | repo_cfg = request.json['repo_cfg'] 822 | 823 | g.states[repo_label] = {} 824 | g.repos[repo_label] = None 825 | g.repo_cfgs[repo_label] = repo_cfg 826 | g.repo_labels[repo_cfg['owner'], repo_cfg['name']] = repo_label 827 | 828 | Thread(target=synchronize, args=[repo_label, g.cfg, repo_cfg, g.logger, 829 | g.gh, g.states, g.repos, g.db, 830 | g.mergeable_que, g.my_username, 831 | g.repo_labels]).start() 832 | return 'OK' 833 | 834 | elif request.json['cmd'] == 'repo_del': 835 | repo_label = request.json['repo_label'] 836 | repo_cfg = g.repo_cfgs[repo_label] 837 | 838 | db_query(g.db, 'DELETE FROM pull WHERE repo = ?', [repo_label]) 839 | db_query(g.db, 'DELETE FROM build_res WHERE repo = ?', [repo_label]) 840 | db_query(g.db, 'DELETE FROM mergeable WHERE repo = ?', [repo_label]) 841 | 842 | del g.states[repo_label] 843 | del g.repos[repo_label] 844 | del g.repo_cfgs[repo_label] 845 | del g.repo_labels[repo_cfg['owner'], repo_cfg['name']] 846 | 847 | return 'OK' 848 | 849 | elif request.json['cmd'] == 'repo_edit': 850 | repo_label = request.json['repo_label'] 851 | repo_cfg = request.json['repo_cfg'] 852 | 853 | assert repo_cfg['owner'] == g.repo_cfgs[repo_label]['owner'] 854 | assert repo_cfg['name'] == g.repo_cfgs[repo_label]['name'] 855 | 856 | g.repo_cfgs[repo_label] = repo_cfg 857 | 858 | return 'OK' 859 | 860 | elif request.json['cmd'] == 'sync_all': 861 | Thread(target=synch_all).start() 862 | 863 | return 'OK' 864 | 865 | return 'Unrecognized command' 866 | 867 | 868 | def start(cfg, states, queue_handler, repo_cfgs, repos, logger, 869 | buildbot_slots, my_username, db, repo_labels, mergeable_que, gh): 870 | env = jinja2.Environment( 871 | loader=jinja2.FileSystemLoader(pkg_resources.resource_filename(__name__, 'html')), # noqa 872 | autoescape=True, 873 | ) 874 | tpls = {} 875 | tpls['index'] = env.get_template('index.html') 876 | tpls['queue'] = env.get_template('queue.html') 877 | tpls['build_res'] = env.get_template('build_res.html') 878 | 879 | g.cfg = cfg 880 | g.states = states 881 | g.queue_handler = queue_handler 882 | g.repo_cfgs = repo_cfgs 883 | g.repos = repos 884 | g.logger = logger.getChild('server') 885 | g.buildbot_slots = buildbot_slots 886 | g.tpls = tpls 887 | g.my_username = my_username 888 | g.db = db 889 | g.repo_labels = repo_labels 890 | g.mergeable_que = mergeable_que 891 | g.gh = gh 892 | 893 | # Synchronize all PR data on startup 894 | if cfg['web'].get('sync_on_start', False): 895 | Thread(target=synch_all).start() 896 | 897 | try: 898 | run(host=cfg['web'].get('host', '0.0.0.0'), 899 | port=cfg['web']['port'], 900 | server='waitress') 901 | except OSError as e: 902 | print(e, file=sys.stderr) 903 | os._exit(1) 904 | -------------------------------------------------------------------------------- /homu/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import github3 3 | import toml 4 | import json 5 | import re 6 | import functools 7 | from enum import IntEnum 8 | from . import utils 9 | from .utils import lazy_debug 10 | from . import action 11 | import logging 12 | from threading import Thread, Lock, Timer 13 | import time 14 | import traceback 15 | import sqlite3 16 | import requests 17 | from contextlib import contextmanager 18 | from itertools import chain 19 | from queue import Queue 20 | import os 21 | import sys 22 | import subprocess 23 | from .git_helper import SSH_KEY_FILE 24 | import shlex 25 | 26 | STATUS_TO_PRIORITY = { 27 | 'success': 0, 28 | 'pending': 1, 29 | 'approved': 2, 30 | '': 3, 31 | 'error': 4, 32 | 'failure': 5, 33 | } 34 | 35 | INTERRUPTED_BY_HOMU_FMT = 'Interrupted by Homu ({})' 36 | INTERRUPTED_BY_HOMU_RE = re.compile(r'Interrupted by Homu \((.+?)\)') 37 | DEFAULT_TEST_TIMEOUT = 3600 * 10 38 | 39 | 40 | class AuthState(IntEnum): 41 | # Higher is more privileged 42 | REVIEWER = 3 43 | TRY = 2 44 | NONE = 1 45 | 46 | 47 | @contextmanager 48 | def buildbot_sess(repo_cfg): 49 | sess = requests.Session() 50 | 51 | sess.post( 52 | repo_cfg['buildbot']['url'] + '/login', 53 | allow_redirects=False, 54 | data={ 55 | 'username': repo_cfg['buildbot']['username'], 56 | 'passwd': repo_cfg['buildbot']['password'], 57 | }) 58 | 59 | yield sess 60 | 61 | sess.get(repo_cfg['buildbot']['url'] + '/logout', allow_redirects=False) 62 | 63 | 64 | db_query_lock = Lock() 65 | 66 | 67 | def db_query(db, *args): 68 | with db_query_lock: 69 | db.execute(*args) 70 | 71 | 72 | class Repository: 73 | treeclosed = -1 74 | gh = None 75 | label = None 76 | db = None 77 | 78 | def __init__(self, gh, repo_label, db): 79 | self.gh = gh 80 | self.repo_label = repo_label 81 | self.db = db 82 | db_query( 83 | db, 84 | 'SELECT treeclosed FROM repos WHERE repo = ?', 85 | [repo_label] 86 | ) 87 | row = db.fetchone() 88 | if row: 89 | self.treeclosed = row[0] 90 | else: 91 | self.treeclosed = -1 92 | 93 | def update_treeclosed(self, value): 94 | self.treeclosed = value 95 | db_query( 96 | self.db, 97 | 'DELETE FROM repos where repo = ?', 98 | [self.repo_label] 99 | ) 100 | if value > 0: 101 | db_query( 102 | self.db, 103 | 'INSERT INTO repos (repo, treeclosed) VALUES (?, ?)', 104 | [self.repo_label, value] 105 | ) 106 | 107 | def __lt__(self, other): 108 | return self.gh < other.gh 109 | 110 | 111 | class PullReqState: 112 | num = 0 113 | priority = 0 114 | rollup = False 115 | title = '' 116 | body = '' 117 | head_ref = '' 118 | base_ref = '' 119 | assignee = '' 120 | delegate = '' 121 | try_choose = None 122 | 123 | def __init__(self, num, head_sha, status, db, repo_label, mergeable_que, 124 | gh, owner, name, label_events, repos): 125 | self.head_advanced('', use_db=False) 126 | 127 | self.num = num 128 | self.head_sha = head_sha 129 | self.status = status 130 | self.db = db 131 | self.repo_label = repo_label 132 | self.mergeable_que = mergeable_que 133 | self.gh = gh 134 | self.owner = owner 135 | self.name = name 136 | self.repos = repos 137 | self.timeout_timer = None 138 | self.test_started = time.time() 139 | self.label_events = label_events 140 | 141 | def head_advanced(self, head_sha, *, use_db=True): 142 | self.head_sha = head_sha 143 | self.approved_by = '' 144 | self.status = '' 145 | self.merge_sha = '' 146 | self.build_res = {} 147 | self.try_ = False 148 | self.mergeable = None 149 | 150 | if use_db: 151 | self.set_status('') 152 | self.set_mergeable(None) 153 | self.init_build_res([]) 154 | 155 | def __repr__(self): 156 | fmt = 'PullReqState:{}/{}#{}(approved_by={}, priority={}, status={})' 157 | return fmt.format( 158 | self.owner, 159 | self.name, 160 | self.num, 161 | self.approved_by, 162 | self.priority, 163 | self.status, 164 | ) 165 | 166 | def sort_key(self): 167 | return [ 168 | STATUS_TO_PRIORITY.get(self.get_status(), -1), 169 | 1 if self.mergeable is False else 0, 170 | 0 if self.approved_by else 1, 171 | 1 if self.rollup else 0, 172 | -self.priority, 173 | self.num, 174 | ] 175 | 176 | def __lt__(self, other): 177 | return self.sort_key() < other.sort_key() 178 | 179 | def get_issue(self): 180 | issue = getattr(self, 'issue', None) 181 | if not issue: 182 | issue = self.issue = self.get_repo().issue(self.num) 183 | return issue 184 | 185 | def add_comment(self, text): 186 | self.get_issue().create_comment(text) 187 | 188 | def change_labels(self, event): 189 | event = self.label_events.get(event.value, {}) 190 | removes = event.get('remove', []) 191 | adds = event.get('add', []) 192 | unless = event.get('unless', []) 193 | if not removes and not adds: 194 | return 195 | 196 | issue = self.get_issue() 197 | labels = {label.name for label in issue.iter_labels()} 198 | if labels.isdisjoint(unless): 199 | labels.difference_update(removes) 200 | labels.update(adds) 201 | issue.replace_labels(list(labels)) 202 | 203 | def set_status(self, status): 204 | self.status = status 205 | if self.timeout_timer: 206 | self.timeout_timer.cancel() 207 | self.timeout_timer = None 208 | 209 | db_query( 210 | self.db, 211 | 'UPDATE pull SET status = ? WHERE repo = ? AND num = ?', 212 | [self.status, self.repo_label, self.num] 213 | ) 214 | 215 | # FIXME: self.try_ should also be saved in the database 216 | if not self.try_: 217 | db_query( 218 | self.db, 219 | 'UPDATE pull SET merge_sha = ? WHERE repo = ? AND num = ?', 220 | [self.merge_sha, self.repo_label, self.num] 221 | ) 222 | 223 | def get_status(self): 224 | if self.status == '' and self.approved_by: 225 | if self.mergeable is not False: 226 | return 'approved' 227 | return self.status 228 | 229 | def set_mergeable(self, mergeable, *, cause=None, que=True): 230 | if mergeable is not None: 231 | self.mergeable = mergeable 232 | 233 | db_query( 234 | self.db, 235 | 'INSERT OR REPLACE INTO mergeable (repo, num, mergeable) VALUES (?, ?, ?)', # noqa 236 | [self.repo_label, self.num, self.mergeable] 237 | ) 238 | else: 239 | if que: 240 | self.mergeable_que.put([self, cause]) 241 | else: 242 | self.mergeable = None 243 | 244 | db_query( 245 | self.db, 246 | 'DELETE FROM mergeable WHERE repo = ? AND num = ?', 247 | [self.repo_label, self.num] 248 | ) 249 | 250 | def init_build_res(self, builders, *, use_db=True): 251 | self.build_res = {x: { 252 | 'res': None, 253 | 'url': '', 254 | } for x in builders} 255 | 256 | if use_db: 257 | db_query( 258 | self.db, 259 | 'DELETE FROM build_res WHERE repo = ? AND num = ?', 260 | [self.repo_label, self.num] 261 | ) 262 | 263 | def set_build_res(self, builder, res, url): 264 | if builder not in self.build_res: 265 | raise Exception('Invalid builder: {}'.format(builder)) 266 | 267 | self.build_res[builder] = { 268 | 'res': res, 269 | 'url': url, 270 | } 271 | 272 | db_query( 273 | self.db, 274 | 'INSERT OR REPLACE INTO build_res (repo, num, builder, res, url, merge_sha) VALUES (?, ?, ?, ?, ?, ?)', # noqa 275 | [ 276 | self.repo_label, 277 | self.num, 278 | builder, 279 | res, 280 | url, 281 | self.merge_sha, 282 | ]) 283 | 284 | def build_res_summary(self): 285 | return ', '.join('{}: {}'.format(builder, data['res']) 286 | for builder, data in self.build_res.items()) 287 | 288 | def get_repo(self): 289 | repo = self.repos[self.repo_label].gh 290 | if not repo: 291 | repo = self.gh.repository(self.owner, self.name) 292 | self.repos[self.repo_label].gh = repo 293 | 294 | assert repo.owner.login == self.owner 295 | assert repo.name == self.name 296 | return repo 297 | 298 | def save(self): 299 | db_query( 300 | self.db, 301 | 'INSERT OR REPLACE INTO pull (repo, num, status, merge_sha, title, body, head_sha, head_ref, base_ref, assignee, approved_by, priority, try_, try_choose, rollup, delegate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', # noqa 302 | [ 303 | self.repo_label, 304 | self.num, 305 | self.status, 306 | self.merge_sha, 307 | self.title, 308 | self.body, 309 | self.head_sha, 310 | self.head_ref, 311 | self.base_ref, 312 | self.assignee, 313 | self.approved_by, 314 | self.priority, 315 | self.try_, 316 | self.try_choose, 317 | self.rollup, 318 | self.delegate, 319 | ]) 320 | 321 | def refresh(self): 322 | issue = self.get_repo().issue(self.num) 323 | 324 | self.title = issue.title 325 | self.body = issue.body 326 | 327 | def fake_merge(self, repo_cfg): 328 | if not repo_cfg.get('linear', False): 329 | return 330 | if repo_cfg.get('autosquash', False): 331 | return 332 | 333 | issue = self.get_issue() 334 | title = issue.title 335 | # We tell github to close the PR via the commit message, but it 336 | # doesn't know that constitutes a merge. Edit the title so that it's 337 | # clearer. 338 | merged_prefix = '[merged] ' 339 | if not title.startswith(merged_prefix): 340 | title = merged_prefix + title 341 | issue.edit(title=title) 342 | 343 | def change_treeclosed(self, value): 344 | self.repos[self.repo_label].update_treeclosed(value) 345 | 346 | def blocked_by_closed_tree(self): 347 | treeclosed = self.repos[self.repo_label].treeclosed 348 | return treeclosed if self.priority < treeclosed else None 349 | 350 | def start_testing(self, timeout): 351 | self.test_started = time.time() # FIXME: Save in the local database 352 | self.set_status('pending') 353 | timer = Timer(timeout, self.timed_out) 354 | timer.start() 355 | self.timeout_timer = timer 356 | 357 | def timed_out(self): 358 | print('* Test timed out: {}'.format(self)) 359 | 360 | self.merge_sha = '' 361 | self.save() 362 | self.set_status('failure') 363 | 364 | desc = 'Test timed out' 365 | utils.github_create_status( 366 | self.get_repo(), 367 | self.head_sha, 368 | 'failure', 369 | '', 370 | desc, 371 | context='homu') 372 | self.add_comment(':boom: {}'.format(desc)) 373 | self.change_labels(action.LabelEvent.TIMED_OUT) 374 | 375 | 376 | def sha_or_blank(sha): 377 | return sha if re.match(r'^[0-9a-f]+$', sha) else '' 378 | 379 | 380 | def verify_auth(username, repo_cfg, state, auth, realtime, my_username): 381 | # In some cases (e.g. non-fully-qualified r+) we recursively talk to 382 | # ourself via a hidden markdown comment in the message. This is so that 383 | # when re-synchronizing after shutdown we can parse these comments and 384 | # still know the SHA for the approval. 385 | # 386 | # So comments from self should always be allowed 387 | if username == my_username: 388 | return True 389 | is_reviewer = False 390 | auth_collaborators = repo_cfg.get('auth_collaborators', False) 391 | if auth_collaborators: 392 | is_reviewer = state.get_repo().is_collaborator(username) 393 | if not is_reviewer: 394 | is_reviewer = username in repo_cfg.get('reviewers', []) 395 | if not is_reviewer: 396 | is_reviewer = username.lower() == state.delegate.lower() 397 | 398 | if is_reviewer: 399 | have_auth = AuthState.REVIEWER 400 | elif username in repo_cfg.get('try_users', []): 401 | have_auth = AuthState.TRY 402 | else: 403 | have_auth = AuthState.NONE 404 | if have_auth >= auth: 405 | return True 406 | else: 407 | if realtime: 408 | reply = '@{}: :key: Insufficient privileges: '.format(username) 409 | if auth == AuthState.REVIEWER: 410 | if auth_collaborators: 411 | reply += 'Collaborator required' 412 | else: 413 | reply += 'Not in reviewers' 414 | elif auth == AuthState.TRY: 415 | reply += 'not in try users' 416 | state.add_comment(reply) 417 | return False 418 | 419 | 420 | def get_words(body, my_username): 421 | return list(chain.from_iterable(re.findall(r'\S+', x) for x in body.splitlines() if '@' + my_username in x)) # noqa 422 | 423 | 424 | def parse_commands(cfg, body, username, repo_cfg, state, my_username, db, 425 | states, *, realtime=False, sha=''): 426 | state_changed = False 427 | 428 | _reviewer_auth_verified = functools.partial( 429 | verify_auth, 430 | username, 431 | repo_cfg, 432 | state, 433 | AuthState.REVIEWER, 434 | realtime, 435 | my_username, 436 | ) 437 | 438 | _try_auth_verified = functools.partial( 439 | verify_auth, 440 | username, 441 | repo_cfg, 442 | state, 443 | AuthState.TRY, 444 | realtime, 445 | my_username, 446 | ) 447 | 448 | words = get_words(body, my_username) 449 | if words[1:] == ["are", "you", "still", "there?"] and realtime: 450 | action.still_here(state) 451 | 452 | # reverse the list, as usually the review status 453 | # is indicated at the end of the comment. 454 | for i, word in reversed(list(enumerate(words))): 455 | found = True 456 | if word == 'r+' or word.startswith('r='): 457 | if not _reviewer_auth_verified(): 458 | continue 459 | 460 | if not sha and i + 1 < len(words): 461 | cur_sha = sha_or_blank(words[i + 1]) 462 | else: 463 | cur_sha = sha 464 | approver = word[len('r='):] if word.startswith('r=') else username 465 | 466 | if not action.review_approved(state, realtime, approver, username, 467 | my_username, cur_sha, states): 468 | continue 469 | 470 | elif word == 'r-': 471 | if not _reviewer_auth_verified(): 472 | continue 473 | action.review_rejected(state, realtime) 474 | 475 | elif word.startswith('p='): 476 | if not _try_auth_verified(): 477 | continue 478 | if not action.set_priority(state, realtime, word[len('p='):], cfg): 479 | continue 480 | 481 | elif word.startswith('delegate='): 482 | if not _reviewer_auth_verified(): 483 | continue 484 | action.delegate_to(state, realtime, word[len('delegate='):]) 485 | 486 | elif word == 'delegate-': 487 | # TODO: why is this a TRY? 488 | if not _try_auth_verified(): 489 | continue 490 | action.delegate_negative(state) 491 | 492 | elif word == 'delegate+': 493 | if not _reviewer_auth_verified(): 494 | continue 495 | action.delegate_positive(state, 496 | state.get_repo(). 497 | pull_request(state.num). 498 | user.login, 499 | realtime) 500 | 501 | elif word == 'retry' and realtime: 502 | if not _try_auth_verified(): 503 | continue 504 | action.retry(state) 505 | 506 | elif word in ['try', 'try-'] and realtime: 507 | if not _try_auth_verified(): 508 | continue 509 | action._try(state, word, realtime, repo_cfg) 510 | 511 | elif word.startswith("try=") and realtime: 512 | if not _try_auth_verified(): 513 | continue 514 | action._try(state, "try", realtime, repo_cfg, word[len('try='):]) 515 | 516 | elif word in ['rollup', 'rollup-']: 517 | if not _try_auth_verified(): 518 | continue 519 | action.rollup(state, word) 520 | 521 | elif word == 'force' and realtime: 522 | if not _try_auth_verified(): 523 | continue 524 | force(repo_cfg, state) 525 | 526 | elif word == 'clean' and realtime: 527 | if not _try_auth_verified(): 528 | continue 529 | action.clean(state) 530 | 531 | elif (word == 'hello?' or word == 'ping') and realtime: 532 | action.hello_or_ping(state) 533 | 534 | elif word.startswith('treeclosed='): 535 | if not _reviewer_auth_verified(): 536 | continue 537 | action.set_treeclosed(state, word.lstrip("treeclosed=")) 538 | 539 | elif word == 'treeclosed-': 540 | if not _reviewer_auth_verified(): 541 | continue 542 | action.treeclosed_negative(state) 543 | 544 | elif 'hooks' in cfg: 545 | # TODO: Can't extract this code to a new function 546 | # because it changes the value of `found`. 547 | hook_found = False 548 | for hook in cfg['hooks']: 549 | hook_cfg = cfg['hooks'][hook] 550 | if hook_cfg['realtime'] and not realtime: 551 | continue 552 | if word == hook or word.startswith('%s=' % hook): 553 | if hook_cfg['access'] == "reviewer": 554 | if not _reviewer_auth_verified(): 555 | continue 556 | else: 557 | if not _try_auth_verified(): 558 | continue 559 | hook_found = True 560 | extra_data = "" 561 | if word.startswith('%s=' % hook): 562 | extra_data = word.split("=")[1] 563 | Thread( 564 | target=handle_hook_response, 565 | args=[state, hook_cfg, body, extra_data] 566 | ).start() 567 | if not hook_found: 568 | found = False 569 | 570 | else: 571 | found = False 572 | 573 | if found: 574 | state_changed = True 575 | words[i] = '' 576 | 577 | return state_changed 578 | 579 | 580 | def force(repo_cfg, state): 581 | if 'buildbot' in repo_cfg: 582 | with buildbot_sess(repo_cfg) as sess: 583 | res = sess.post( 584 | repo_cfg['buildbot']['url'] + '/builders/_selected/stopselected', # noqa 585 | allow_redirects=False, 586 | data={ 587 | 'selected': repo_cfg['buildbot']['builders'], 588 | 'comments': INTERRUPTED_BY_HOMU_FMT.format(int(time.time())), # noqa 589 | }) 590 | if 'authzfail' in res.text: 591 | err = 'Authorization failed' 592 | else: 593 | mat = re.search('(?s)
(.*?)
', res.text) 594 | if mat: 595 | err = mat.group(1).strip() 596 | if not err: 597 | err = 'Unknown error' 598 | else: 599 | err = '' 600 | if err: 601 | state.add_comment( 602 | ':bomb: Buildbot returned an error: `{}`'.format(err) 603 | ) 604 | 605 | 606 | def handle_hook_response(state, hook_cfg, body, extra_data): 607 | post_data = {} 608 | post_data["pull"] = state.num 609 | post_data["body"] = body 610 | post_data["extra_data"] = extra_data 611 | print(post_data) 612 | response = requests.post(hook_cfg['endpoint'], json=post_data) 613 | print(response.text) 614 | 615 | # We only post a response if we're configured to have a response 616 | # non-realtime hooks cannot post 617 | if hook_cfg['has_response'] and hook_cfg['realtime']: 618 | state.add_comment(response.text) 619 | 620 | 621 | def git_push(git_cmd, branch, state): 622 | merge_sha = subprocess.check_output(git_cmd('rev-parse', 'HEAD')).decode('ascii').strip() # noqa 623 | 624 | if utils.silent_call(git_cmd('push', '-f', 'origin', branch)): 625 | utils.logged_call(git_cmd('branch', '-f', 'homu-tmp', branch)) 626 | utils.logged_call(git_cmd('push', '-f', 'origin', 'homu-tmp')) 627 | 628 | def inner(): 629 | utils.github_create_status( 630 | state.get_repo(), 631 | merge_sha, 632 | 'success', 633 | '', 634 | 'Branch protection bypassed', 635 | context='homu', 636 | ) 637 | 638 | def fail(err): 639 | state.add_comment( 640 | ':boom: Unable to create a status for {} ({})' 641 | .format(merge_sha, err) 642 | ) 643 | 644 | utils.retry_until(inner, fail, state) 645 | 646 | utils.logged_call(git_cmd('push', '-f', 'origin', branch)) 647 | 648 | return merge_sha 649 | 650 | 651 | def init_local_git_cmds(repo_cfg, git_cfg): 652 | fpath = 'cache/{}/{}'.format(repo_cfg['owner'], repo_cfg['name']) 653 | url = 'git@github.com:{}/{}.git'.format(repo_cfg['owner'], repo_cfg['name']) # noqa 654 | 655 | if not os.path.exists(SSH_KEY_FILE): 656 | os.makedirs(os.path.dirname(SSH_KEY_FILE), exist_ok=True) 657 | with open(SSH_KEY_FILE, 'w') as fp: 658 | fp.write(git_cfg['ssh_key']) 659 | os.chmod(SSH_KEY_FILE, 0o600) 660 | 661 | if not os.path.exists(fpath): 662 | utils.logged_call(['git', 'init', fpath]) 663 | utils.logged_call(['git', '-C', fpath, 'remote', 'add', 'origin', url]) # noqa 664 | 665 | return lambda *args: ['git', '-C', fpath] + list(args) 666 | 667 | 668 | def branch_equal_to_merge(git_cmd, state, branch): 669 | utils.logged_call(git_cmd('fetch', 'origin', 670 | 'pull/{}/merge'.format(state.num))) 671 | return utils.silent_call(git_cmd('diff', '--quiet', 'FETCH_HEAD', branch)) == 0 # noqa 672 | 673 | 674 | def create_merge(state, repo_cfg, branch, logger, git_cfg, 675 | ensure_merge_equal=False): 676 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 677 | 678 | state.refresh() 679 | 680 | lazy_debug(logger, 681 | lambda: "create_merge: attempting merge {} into {} on {!r}" 682 | .format(state.head_sha, branch, state.get_repo())) 683 | 684 | merge_msg = 'Auto merge of #{} - {}, r={}\n\n{}\n\n{}'.format( 685 | state.num, 686 | state.head_ref, 687 | '' if state.try_ else state.approved_by, 688 | state.title, 689 | state.body) 690 | 691 | desc = 'Merge conflict' 692 | 693 | if git_cfg['local_git']: 694 | 695 | git_cmd = init_local_git_cmds(repo_cfg, git_cfg) 696 | 697 | utils.logged_call(git_cmd('fetch', 'origin', state.base_ref, 698 | 'pull/{}/head'.format(state.num))) 699 | utils.silent_call(git_cmd('rebase', '--abort')) 700 | utils.silent_call(git_cmd('merge', '--abort')) 701 | 702 | if repo_cfg.get('linear', False): 703 | utils.logged_call( 704 | git_cmd('checkout', '-B', branch, state.head_sha)) 705 | try: 706 | args = [base_sha] 707 | if repo_cfg.get('autosquash', False): 708 | args += ['-i', '--autosquash'] 709 | utils.logged_call(git_cmd('-c', 710 | 'user.name=' + git_cfg['name'], 711 | '-c', 712 | 'user.email=' + git_cfg['email'], 713 | 'rebase', 714 | *args)) 715 | except subprocess.CalledProcessError: 716 | if repo_cfg.get('autosquash', False): 717 | utils.silent_call(git_cmd('rebase', '--abort')) 718 | if utils.silent_call(git_cmd('rebase', base_sha)) == 0: 719 | desc = 'Auto-squashing failed' 720 | else: 721 | ap = '' if state.try_ else state.approved_by 722 | text = '\nCloses: #{}\nApproved by: {}'.format(state.num, ap) 723 | msg_code = 'cat && echo {}'.format(shlex.quote(text)) 724 | env_code = 'export GIT_COMMITTER_NAME={} && export GIT_COMMITTER_EMAIL={} && unset GIT_COMMITTER_DATE'.format(shlex.quote(git_cfg['name']), shlex.quote(git_cfg['email'])) # noqa 725 | utils.logged_call(git_cmd('filter-branch', '-f', 726 | '--msg-filter', msg_code, 727 | '--env-filter', env_code, 728 | '{}..'.format(base_sha))) 729 | 730 | if ensure_merge_equal: 731 | if not branch_equal_to_merge(git_cmd, state, branch): 732 | return '' 733 | 734 | return git_push(git_cmd, branch, state) 735 | else: 736 | utils.logged_call(git_cmd( 737 | 'checkout', 738 | '-B', 739 | 'homu-tmp', 740 | state.head_sha)) 741 | 742 | ok = True 743 | if repo_cfg.get('autosquash', False): 744 | try: 745 | merge_base_sha = subprocess.check_output( 746 | git_cmd( 747 | 'merge-base', 748 | base_sha, 749 | state.head_sha)).decode('ascii').strip() 750 | utils.logged_call(git_cmd( 751 | '-c', 752 | 'user.name=' + git_cfg['name'], 753 | '-c', 754 | 'user.email=' + git_cfg['email'], 755 | 'rebase', 756 | '-i', 757 | '--autosquash', 758 | '--onto', 759 | merge_base_sha, base_sha)) 760 | except subprocess.CalledProcessError: 761 | desc = 'Auto-squashing failed' 762 | ok = False 763 | 764 | if ok: 765 | utils.logged_call(git_cmd('checkout', '-B', branch, base_sha)) 766 | try: 767 | utils.logged_call(git_cmd( 768 | '-c', 769 | 'user.name=' + git_cfg['name'], 770 | '-c', 771 | 'user.email=' + git_cfg['email'], 772 | 'merge', 773 | 'heads/homu-tmp', 774 | '--no-ff', 775 | '-m', 776 | merge_msg)) 777 | except subprocess.CalledProcessError: 778 | pass 779 | else: 780 | if ensure_merge_equal: 781 | if not branch_equal_to_merge(git_cmd, state, branch): 782 | return '' 783 | 784 | return git_push(git_cmd, branch, state) 785 | else: 786 | if repo_cfg.get('linear', False) or repo_cfg.get('autosquash', False): 787 | raise RuntimeError('local_git must be turned on to use this feature') # noqa 788 | 789 | # if we're merging using the GitHub API, we have no way to predict 790 | # with certainty what the final result will be so make sure the caller 791 | # isn't asking us to keep any promises (see also discussions at 792 | # https://github.com/servo/homu/pull/57) 793 | assert ensure_merge_equal is False 794 | 795 | if branch != state.base_ref: 796 | utils.github_set_ref( 797 | state.get_repo(), 798 | 'heads/' + branch, 799 | base_sha, 800 | force=True, 801 | ) 802 | 803 | try: 804 | merge_commit = state.get_repo().merge( 805 | branch, 806 | state.head_sha, 807 | merge_msg) 808 | except github3.models.GitHubError as e: 809 | if e.code != 409: 810 | raise 811 | else: 812 | return merge_commit.sha if merge_commit else '' 813 | 814 | state.set_status('error') 815 | utils.github_create_status( 816 | state.get_repo(), 817 | state.head_sha, 818 | 'error', 819 | '', 820 | desc, 821 | context='homu') 822 | 823 | state.add_comment(':lock: ' + desc) 824 | state.change_labels(action.LabelEvent.CONFLICT) 825 | 826 | return '' 827 | 828 | 829 | def pull_is_rebased(state, repo_cfg, git_cfg, base_sha): 830 | assert git_cfg['local_git'] 831 | git_cmd = init_local_git_cmds(repo_cfg, git_cfg) 832 | 833 | utils.logged_call(git_cmd('fetch', 'origin', state.base_ref, 834 | 'pull/{}/head'.format(state.num))) 835 | 836 | return utils.silent_call(git_cmd('merge-base', '--is-ancestor', 837 | base_sha, state.head_sha)) == 0 838 | 839 | 840 | # We could fetch this from GitHub instead, but that API is being deprecated: 841 | # https://developer.github.com/changes/2013-04-25-deprecating-merge-commit-sha/ 842 | def get_github_merge_sha(state, repo_cfg, git_cfg): 843 | assert git_cfg['local_git'] 844 | git_cmd = init_local_git_cmds(repo_cfg, git_cfg) 845 | 846 | if state.mergeable is not True: 847 | return None 848 | 849 | utils.logged_call(git_cmd('fetch', 'origin', 850 | 'pull/{}/merge'.format(state.num))) 851 | 852 | return subprocess.check_output(git_cmd('rev-parse', 'FETCH_HEAD')).decode('ascii').strip() # noqa 853 | 854 | 855 | def do_exemption_merge(state, logger, repo_cfg, git_cfg, url, check_merge, 856 | reason): 857 | 858 | try: 859 | merge_sha = create_merge( 860 | state, 861 | repo_cfg, 862 | state.base_ref, 863 | logger, 864 | git_cfg, 865 | check_merge) 866 | except subprocess.CalledProcessError: 867 | print('* Unable to create a merge commit for the exempted PR: {}'.format(state)) # noqa 868 | traceback.print_exc() 869 | return False 870 | 871 | if not merge_sha: 872 | return False 873 | 874 | desc = 'Test exempted' 875 | 876 | state.set_status('success') 877 | utils.github_create_status(state.get_repo(), state.head_sha, 'success', 878 | url, desc, context='homu') 879 | state.add_comment(':zap: {}: {}.'.format(desc, reason)) 880 | state.change_labels(action.LabelEvent.EXEMPTED) 881 | 882 | state.merge_sha = merge_sha 883 | state.save() 884 | 885 | state.fake_merge(repo_cfg) 886 | return True 887 | 888 | 889 | def try_travis_exemption(state, logger, repo_cfg, git_cfg): 890 | 891 | travis_info = None 892 | for info in utils.github_iter_statuses(state.get_repo(), state.head_sha): 893 | if info.context == 'continuous-integration/travis-ci/pr': 894 | travis_info = info 895 | break 896 | 897 | if travis_info is None or travis_info.state != 'success': 898 | return False 899 | 900 | mat = re.search('/builds/([0-9]+)$', travis_info.target_url) 901 | if not mat: 902 | return False 903 | 904 | url = 'https://api.travis-ci.org/{}/{}/builds/{}'.format(state.owner, 905 | state.name, 906 | mat.group(1)) 907 | try: 908 | res = requests.get(url) 909 | except Exception as ex: 910 | print('* Unable to gather build info from Travis CI: {}'.format(ex)) 911 | return False 912 | 913 | travis_sha = json.loads(res.text)['commit'] 914 | travis_commit = state.get_repo().commit(travis_sha) 915 | 916 | if not travis_commit: 917 | return False 918 | 919 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 920 | 921 | if (travis_commit.parents[0]['sha'] == base_sha and 922 | travis_commit.parents[1]['sha'] == state.head_sha): 923 | # make sure we check against the github merge sha before pushing 924 | return do_exemption_merge(state, logger, repo_cfg, git_cfg, 925 | travis_info.target_url, True, 926 | "merge already tested by Travis CI") 927 | 928 | return False 929 | 930 | 931 | def try_status_exemption(state, logger, repo_cfg, git_cfg): 932 | 933 | # If all the builders are status-based, then we can do some checks to 934 | # exempt testing under the following cases: 935 | # 1. The PR head commit has the equivalent statuses set to 'success' and 936 | # it is fully rebased on the HEAD of the target base ref. 937 | # 2. The PR head and merge commits have the equivalent statuses set to 938 | # state 'success' and the merge commit's first parent is the HEAD of 939 | # the target base ref. 940 | 941 | if not git_cfg['local_git']: 942 | raise RuntimeError('local_git is required to use status exemption') 943 | 944 | statuses_all = set() 945 | 946 | # equivalence dict: pr context --> auto context 947 | status_equivalences = {} 948 | 949 | for key, value in repo_cfg['status'].items(): 950 | context = value.get('context') 951 | pr_context = value.get('pr_context', context) 952 | if context is not None: 953 | statuses_all.add(context) 954 | status_equivalences[pr_context] = context 955 | 956 | assert len(statuses_all) > 0 957 | 958 | # let's first check that all the statuses we want are set to success 959 | statuses_pass = set() 960 | for info in utils.github_iter_statuses(state.get_repo(), state.head_sha): 961 | if info.context in status_equivalences and info.state == 'success': 962 | statuses_pass.add(status_equivalences[info.context]) 963 | 964 | if statuses_all != statuses_pass: 965 | return False 966 | 967 | # is the PR fully rebased? 968 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 969 | if pull_is_rebased(state, repo_cfg, git_cfg, base_sha): 970 | return do_exemption_merge(state, logger, repo_cfg, git_cfg, '', False, 971 | "pull fully rebased and already tested") 972 | 973 | # check if we can use the github merge sha as proof 974 | merge_sha = get_github_merge_sha(state, repo_cfg, git_cfg) 975 | if merge_sha is None: 976 | return False 977 | 978 | statuses_merge_pass = set() 979 | for info in utils.github_iter_statuses(state.get_repo(), merge_sha): 980 | if info.context in status_equivalences and info.state == 'success': 981 | statuses_merge_pass.add(status_equivalences[info.context]) 982 | 983 | merge_commit = state.get_repo().commit(merge_sha) 984 | if (statuses_all == statuses_merge_pass and 985 | merge_commit.parents[0]['sha'] == base_sha and 986 | merge_commit.parents[1]['sha'] == state.head_sha): 987 | # make sure we check against the github merge sha before pushing 988 | return do_exemption_merge(state, logger, repo_cfg, git_cfg, '', True, 989 | "merge already tested") 990 | 991 | return False 992 | 993 | 994 | def start_build(state, repo_cfgs, buildbot_slots, logger, db, git_cfg): 995 | if buildbot_slots[0]: 996 | return True 997 | 998 | lazy_debug(logger, lambda: "start_build on {!r}".format(state.get_repo())) 999 | 1000 | assert state.head_sha == state.get_repo().pull_request(state.num).head.sha 1001 | 1002 | repo_cfg = repo_cfgs[state.repo_label] 1003 | 1004 | builders = [] 1005 | branch = 'auto' 1006 | if state.try_: 1007 | if state.try_choose: 1008 | branch = "try-%s" % state.try_choose 1009 | else: 1010 | branch = "try" 1011 | branch = repo_cfg.get('branch', {}).get(branch, branch) 1012 | can_try_travis_exemption = False 1013 | 1014 | only_status_builders = True 1015 | any_buildbot_builders = False 1016 | if 'buildbot' in repo_cfg: 1017 | if state.try_: 1018 | if state.try_choose: 1019 | builders += ( 1020 | repo_cfg['buildbot']['try_choosers'] 1021 | .get(state.try_choose, []) 1022 | ) 1023 | else: 1024 | builders += repo_cfg['buildbot']['try_builders'] 1025 | else: 1026 | builders += repo_cfg['buildbot']['builders'] 1027 | if builders: 1028 | only_status_builders = False 1029 | any_buildbot_builders = True 1030 | if 'travis' in repo_cfg: 1031 | builders += ['travis'] 1032 | only_status_builders = False 1033 | if 'status' in repo_cfg: 1034 | found_travis_context = False 1035 | for key, value in repo_cfg['status'].items(): 1036 | context = value.get('context') 1037 | if context is not None: 1038 | if state.try_ and not value.get('try', True): 1039 | # Skip this builder for tries. 1040 | continue 1041 | builders += ['status-' + key] 1042 | # We have an optional fast path if the Travis test passed 1043 | # for a given commit and master is unchanged, we can do 1044 | # a direct push. 1045 | if context == 'continuous-integration/travis-ci/push': 1046 | found_travis_context = True 1047 | 1048 | if found_travis_context and len(builders) == 1: 1049 | can_try_travis_exemption = True 1050 | if 'checks' in repo_cfg: 1051 | builders += ['checks-' + key for key, value in repo_cfg['checks'].items() if 'name' in value] # noqa 1052 | only_status_builders = False 1053 | 1054 | if len(builders) == 0: 1055 | raise RuntimeError('Invalid configuration') 1056 | 1057 | lazy_debug(logger, lambda: "start_build: builders={!r}".format(builders)) 1058 | 1059 | if (only_status_builders and state.approved_by and 1060 | repo_cfg.get('status_based_exemption', False)): 1061 | if can_try_travis_exemption: 1062 | if try_travis_exemption(state, logger, repo_cfg, git_cfg): 1063 | return True 1064 | if try_status_exemption(state, logger, repo_cfg, git_cfg): 1065 | return True 1066 | 1067 | merge_sha = create_merge(state, repo_cfg, branch, logger, git_cfg) 1068 | lazy_debug(logger, lambda: "start_build: merge_sha={}".format(merge_sha)) 1069 | if not merge_sha: 1070 | return False 1071 | 1072 | state.init_build_res(builders) 1073 | state.merge_sha = merge_sha 1074 | 1075 | state.save() 1076 | 1077 | if any_buildbot_builders: 1078 | buildbot_slots[0] = state.merge_sha 1079 | 1080 | logger.info('Starting build of {}/{}#{} on {}: {}'.format( 1081 | state.owner, 1082 | state.name, 1083 | state.num, 1084 | branch, 1085 | state.merge_sha)) 1086 | 1087 | timeout = repo_cfg.get('timeout', DEFAULT_TEST_TIMEOUT) 1088 | state.start_testing(timeout) 1089 | 1090 | desc = '{} commit {} with merge {}...'.format( 1091 | 'Trying' if state.try_ else 'Testing', 1092 | state.head_sha, 1093 | state.merge_sha, 1094 | ) 1095 | utils.github_create_status( 1096 | state.get_repo(), 1097 | state.head_sha, 1098 | 'pending', 1099 | '', 1100 | desc, 1101 | context='homu') 1102 | 1103 | state.add_comment(':hourglass: ' + desc) 1104 | 1105 | return True 1106 | 1107 | 1108 | def start_rebuild(state, repo_cfgs): 1109 | repo_cfg = repo_cfgs[state.repo_label] 1110 | 1111 | if 'buildbot' not in repo_cfg or not state.build_res: 1112 | return False 1113 | 1114 | builders = [] 1115 | succ_builders = [] 1116 | 1117 | for builder, info in state.build_res.items(): 1118 | if not info['url']: 1119 | return False 1120 | 1121 | if info['res']: 1122 | succ_builders.append([builder, info['url']]) 1123 | else: 1124 | builders.append([builder, info['url']]) 1125 | 1126 | if not builders or not succ_builders: 1127 | return False 1128 | 1129 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 1130 | _parents = state.get_repo().commit(state.merge_sha).parents 1131 | parent_shas = [x['sha'] for x in _parents] 1132 | 1133 | if base_sha not in parent_shas: 1134 | return False 1135 | 1136 | utils.github_set_ref( 1137 | state.get_repo(), 1138 | 'tags/homu-tmp', 1139 | state.merge_sha, 1140 | force=True) 1141 | 1142 | builders.sort() 1143 | succ_builders.sort() 1144 | 1145 | with buildbot_sess(repo_cfg) as sess: 1146 | for builder, url in builders: 1147 | res = sess.post(url + '/rebuild', allow_redirects=False, data={ 1148 | 'useSourcestamp': 'exact', 1149 | 'comments': 'Initiated by Homu', 1150 | }) 1151 | 1152 | if 'authzfail' in res.text: 1153 | err = 'Authorization failed' 1154 | elif builder in res.text: 1155 | err = '' 1156 | else: 1157 | mat = re.search('(.+?)', res.text) 1158 | err = mat.group(1) if mat else 'Unknown error' 1159 | 1160 | if err: 1161 | state.add_comment(':bomb: Failed to start rebuilding: `{}`'.format(err)) # noqa 1162 | return False 1163 | 1164 | timeout = repo_cfg.get('timeout', DEFAULT_TEST_TIMEOUT) 1165 | state.start_testing(timeout) 1166 | 1167 | msg_1 = 'Previous build results' 1168 | msg_2 = ' for {}'.format(', '.join('[{}]({})'.format(builder, url) for builder, url in succ_builders)) # noqa 1169 | msg_3 = ' are reusable. Rebuilding' 1170 | msg_4 = ' only {}'.format(', '.join('[{}]({})'.format(builder, url) for builder, url in builders)) # noqa 1171 | 1172 | utils.github_create_status( 1173 | state.get_repo(), 1174 | state.head_sha, 1175 | 'pending', 1176 | '', 1177 | '{}{}...'.format(msg_1, msg_3), 1178 | context='homu') 1179 | 1180 | state.add_comment(':zap: {}{}{}{}...'.format(msg_1, msg_2, msg_3, msg_4)) 1181 | 1182 | return True 1183 | 1184 | 1185 | def start_build_or_rebuild(state, repo_cfgs, *args): 1186 | if start_rebuild(state, repo_cfgs): 1187 | return True 1188 | 1189 | return start_build(state, repo_cfgs, *args) 1190 | 1191 | 1192 | def process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, 1193 | git_cfg): 1194 | for repo_label, repo in repos.items(): 1195 | repo_states = sorted(states[repo_label].values()) 1196 | 1197 | for state in repo_states: 1198 | lazy_debug(logger, lambda: "process_queue: state={!r}, building {}" 1199 | .format(state, repo_label)) 1200 | if state.priority < repo.treeclosed: 1201 | continue 1202 | if state.status == 'pending' and not state.try_: 1203 | break 1204 | 1205 | elif state.status == 'success' and hasattr(state, 'fake_merge_sha'): # noqa 1206 | break 1207 | 1208 | elif state.status == '' and state.approved_by: 1209 | if start_build_or_rebuild(state, repo_cfgs, buildbot_slots, 1210 | logger, db, git_cfg): 1211 | return 1212 | 1213 | elif state.status == 'success' and state.try_ and state.approved_by: # noqa 1214 | state.try_ = False 1215 | 1216 | state.save() 1217 | 1218 | if start_build(state, repo_cfgs, buildbot_slots, logger, db, 1219 | git_cfg): 1220 | return 1221 | 1222 | for state in repo_states: 1223 | if state.status == '' and state.try_: 1224 | if start_build(state, repo_cfgs, buildbot_slots, logger, db, 1225 | git_cfg): 1226 | return 1227 | 1228 | 1229 | def fetch_mergeability(mergeable_que): 1230 | re_pull_num = re.compile('(?i)merge (?:of|pull request) #([0-9]+)') 1231 | 1232 | while True: 1233 | try: 1234 | state, cause = mergeable_que.get() 1235 | 1236 | if state.status == 'success': 1237 | continue 1238 | 1239 | pull_request = state.get_repo().pull_request(state.num) 1240 | if pull_request is None or pull_request.mergeable is None: 1241 | time.sleep(5) 1242 | pull_request = state.get_repo().pull_request(state.num) 1243 | mergeable = pull_request is not None and pull_request.mergeable 1244 | 1245 | if state.mergeable is True and mergeable is False: 1246 | if cause: 1247 | mat = re_pull_num.search(cause['title']) 1248 | 1249 | if mat: 1250 | issue_or_commit = '#' + mat.group(1) 1251 | else: 1252 | issue_or_commit = cause['sha'] 1253 | else: 1254 | issue_or_commit = '' 1255 | 1256 | _blame = '' 1257 | if issue_or_commit: 1258 | _blame = ' (presumably {})'.format(issue_or_commit) 1259 | state.add_comment(':umbrella: The latest upstream changes{} made this pull request unmergeable. Please resolve the merge conflicts.'.format( # noqa 1260 | _blame 1261 | )) 1262 | state.change_labels(action.LabelEvent.CONFLICT) 1263 | 1264 | state.set_mergeable(mergeable, que=False) 1265 | 1266 | except Exception: 1267 | print('* Error while fetching mergeability') 1268 | traceback.print_exc() 1269 | 1270 | finally: 1271 | mergeable_que.task_done() 1272 | 1273 | 1274 | def synchronize(repo_label, cfg, repo_cfg, logger, gh, states, repos, db, mergeable_que, my_username, repo_labels): # noqa 1275 | logger.info('Synchronizing {}...'.format(repo_label)) 1276 | 1277 | repo = gh.repository(repo_cfg['owner'], repo_cfg['name']) 1278 | 1279 | db_query(db, 'DELETE FROM pull WHERE repo = ?', [repo_label]) 1280 | db_query(db, 'DELETE FROM build_res WHERE repo = ?', [repo_label]) 1281 | db_query(db, 'DELETE FROM mergeable WHERE repo = ?', [repo_label]) 1282 | 1283 | saved_states = {} 1284 | for num, state in states[repo_label].items(): 1285 | saved_states[num] = { 1286 | 'merge_sha': state.merge_sha, 1287 | 'build_res': state.build_res, 1288 | } 1289 | 1290 | states[repo_label] = {} 1291 | repos[repo_label] = Repository(repo, repo_label, db) 1292 | 1293 | for pull in repo.iter_pulls(state='open'): 1294 | db_query( 1295 | db, 1296 | 'SELECT status FROM pull WHERE repo = ? AND num = ?', 1297 | [repo_label, pull.number]) 1298 | row = db.fetchone() 1299 | if row: 1300 | status = row[0] 1301 | else: 1302 | status = '' 1303 | for info in utils.github_iter_statuses(repo, pull.head.sha): 1304 | if info.context == 'homu': 1305 | status = info.state 1306 | break 1307 | 1308 | state = PullReqState(pull.number, pull.head.sha, status, db, repo_label, mergeable_que, gh, repo_cfg['owner'], repo_cfg['name'], repo_cfg.get('labels', {}), repos) # noqa 1309 | state.title = pull.title 1310 | state.body = pull.body 1311 | state.head_ref = pull.head.repo[0] + ':' + pull.head.ref 1312 | state.base_ref = pull.base.ref 1313 | state.set_mergeable(None) 1314 | state.assignee = pull.assignee.login if pull.assignee else '' 1315 | 1316 | for comment in pull.iter_comments(): 1317 | if comment.original_commit_id == pull.head.sha: 1318 | parse_commands( 1319 | cfg, 1320 | comment.body, 1321 | comment.user.login, 1322 | repo_cfg, 1323 | state, 1324 | my_username, 1325 | db, 1326 | states, 1327 | sha=comment.original_commit_id, 1328 | ) 1329 | 1330 | for comment in pull.iter_issue_comments(): 1331 | parse_commands( 1332 | cfg, 1333 | comment.body, 1334 | comment.user.login, 1335 | repo_cfg, 1336 | state, 1337 | my_username, 1338 | db, 1339 | states, 1340 | ) 1341 | 1342 | saved_state = saved_states.get(pull.number) 1343 | if saved_state: 1344 | for key, val in saved_state.items(): 1345 | setattr(state, key, val) 1346 | 1347 | state.save() 1348 | 1349 | states[repo_label][pull.number] = state 1350 | 1351 | logger.info('Done synchronizing {}!'.format(repo_label)) 1352 | 1353 | 1354 | def arguments(): 1355 | parser = argparse.ArgumentParser( 1356 | description='A bot that integrates with GitHub and your favorite ' 1357 | 'continuous integration service') 1358 | parser.add_argument( 1359 | '-v', 1360 | '--verbose', 1361 | action='store_true', 1362 | help='Enable more verbose logging') 1363 | parser.add_argument( 1364 | '-c', 1365 | '--config', 1366 | action='store', 1367 | help='Path to cfg.toml', 1368 | default='cfg.toml') 1369 | 1370 | return parser.parse_args() 1371 | 1372 | 1373 | def main(): 1374 | args = arguments() 1375 | 1376 | logger = logging.getLogger('homu') 1377 | logger.setLevel(logging.DEBUG if args.verbose else logging.INFO) 1378 | logger.addHandler(logging.StreamHandler()) 1379 | 1380 | if sys.getfilesystemencoding() == 'ascii': 1381 | logger.info('You need to set a locale compatible with unicode or homu will choke on Unicode in PR descriptions/titles. See http://stackoverflow.com/a/27931669') # noqa 1382 | 1383 | try: 1384 | with open(args.config) as fp: 1385 | cfg = toml.loads(fp.read()) 1386 | except FileNotFoundError: 1387 | # Fall back to cfg.json only if we're using the defaults 1388 | if args.config == 'cfg.toml': 1389 | with open('cfg.json') as fp: 1390 | cfg = json.loads(fp.read()) 1391 | else: 1392 | raise 1393 | 1394 | gh = github3.login(token=cfg['github']['access_token']) 1395 | user = gh.user() 1396 | cfg_git = cfg.get('git', {}) 1397 | user_email = cfg_git.get('email') 1398 | if user_email is None: 1399 | try: 1400 | user_email = [x for x in gh.iter_emails() if x['primary']][0]['email'] # noqa 1401 | except IndexError: 1402 | raise RuntimeError('Primary email not set, or "user" scope not granted') # noqa 1403 | user_name = cfg_git.get('name', user.name if user.name else user.login) 1404 | 1405 | states = {} 1406 | repos = {} 1407 | repo_cfgs = {} 1408 | buildbot_slots = [''] 1409 | my_username = user.login 1410 | repo_labels = {} 1411 | mergeable_que = Queue() 1412 | git_cfg = { 1413 | 'name': user_name, 1414 | 'email': user_email, 1415 | 'ssh_key': cfg_git.get('ssh_key', ''), 1416 | 'local_git': cfg_git.get('local_git', False), 1417 | } 1418 | 1419 | db_file = cfg.get('db', {}).get('file', 'main.db') 1420 | db_conn = sqlite3.connect(db_file, 1421 | check_same_thread=False, 1422 | isolation_level=None) 1423 | db = db_conn.cursor() 1424 | 1425 | db_query(db, '''CREATE TABLE IF NOT EXISTS pull ( 1426 | repo TEXT NOT NULL, 1427 | num INTEGER NOT NULL, 1428 | status TEXT NOT NULL, 1429 | merge_sha TEXT, 1430 | title TEXT, 1431 | body TEXT, 1432 | head_sha TEXT, 1433 | head_ref TEXT, 1434 | base_ref TEXT, 1435 | assignee TEXT, 1436 | approved_by TEXT, 1437 | priority INTEGER, 1438 | try_ INTEGER, 1439 | try_choose TEXT, 1440 | rollup INTEGER, 1441 | delegate TEXT, 1442 | UNIQUE (repo, num) 1443 | )''') 1444 | 1445 | db_query(db, '''CREATE TABLE IF NOT EXISTS build_res ( 1446 | repo TEXT NOT NULL, 1447 | num INTEGER NOT NULL, 1448 | builder TEXT NOT NULL, 1449 | res INTEGER, 1450 | url TEXT NOT NULL, 1451 | merge_sha TEXT NOT NULL, 1452 | UNIQUE (repo, num, builder) 1453 | )''') 1454 | 1455 | db_query(db, '''CREATE TABLE IF NOT EXISTS mergeable ( 1456 | repo TEXT NOT NULL, 1457 | num INTEGER NOT NULL, 1458 | mergeable INTEGER NOT NULL, 1459 | UNIQUE (repo, num) 1460 | )''') 1461 | db_query(db, '''CREATE TABLE IF NOT EXISTS repos ( 1462 | repo TEXT NOT NULL, 1463 | treeclosed INTEGER NOT NULL, 1464 | UNIQUE (repo) 1465 | )''') 1466 | for repo_label, repo_cfg in cfg['repo'].items(): 1467 | repo_cfgs[repo_label] = repo_cfg 1468 | repo_labels[repo_cfg['owner'], repo_cfg['name']] = repo_label 1469 | 1470 | repo_states = {} 1471 | repos[repo_label] = Repository(None, repo_label, db) 1472 | 1473 | db_query( 1474 | db, 1475 | 'SELECT num, head_sha, status, title, body, head_ref, base_ref, assignee, approved_by, priority, try_, try_choose, rollup, delegate, merge_sha FROM pull WHERE repo = ?', # noqa 1476 | [repo_label]) 1477 | for num, head_sha, status, title, body, head_ref, base_ref, assignee, approved_by, priority, try_, try_choose, rollup, delegate, merge_sha in db.fetchall(): # noqa 1478 | state = PullReqState(num, head_sha, status, db, repo_label, mergeable_que, gh, repo_cfg['owner'], repo_cfg['name'], repo_cfg.get('labels', {}), repos) # noqa 1479 | state.title = title 1480 | state.body = body 1481 | state.head_ref = head_ref 1482 | state.base_ref = base_ref 1483 | state.assignee = assignee 1484 | 1485 | state.approved_by = approved_by 1486 | state.priority = int(priority) 1487 | state.try_ = bool(try_) 1488 | state.try_choose = try_choose 1489 | state.rollup = bool(rollup) 1490 | state.delegate = delegate 1491 | builders = [] 1492 | if merge_sha: 1493 | if 'buildbot' in repo_cfg: 1494 | builders += repo_cfg['buildbot']['builders'] 1495 | if 'travis' in repo_cfg: 1496 | builders += ['travis'] 1497 | if 'status' in repo_cfg: 1498 | builders += ['status-' + key for key, value in repo_cfg['status'].items() if 'context' in value] # noqa 1499 | if 'checks' in repo_cfg: 1500 | builders += ['checks-' + key for key, value in repo_cfg['checks'].items() if 'name' in value] # noqa 1501 | if len(builders) == 0: 1502 | raise RuntimeError('Invalid configuration') 1503 | 1504 | state.init_build_res(builders, use_db=False) 1505 | state.merge_sha = merge_sha 1506 | 1507 | elif state.status == 'pending': 1508 | # FIXME: There might be a better solution 1509 | state.status = '' 1510 | 1511 | state.save() 1512 | 1513 | repo_states[num] = state 1514 | 1515 | states[repo_label] = repo_states 1516 | 1517 | db_query( 1518 | db, 1519 | 'SELECT repo, num, builder, res, url, merge_sha FROM build_res') 1520 | for repo_label, num, builder, res, url, merge_sha in db.fetchall(): 1521 | try: 1522 | state = states[repo_label][num] 1523 | if builder not in state.build_res: 1524 | raise KeyError 1525 | if state.merge_sha != merge_sha: 1526 | raise KeyError 1527 | except KeyError: 1528 | db_query( 1529 | db, 1530 | 'DELETE FROM build_res WHERE repo = ? AND num = ? AND builder = ?', # noqa 1531 | [repo_label, num, builder]) 1532 | continue 1533 | 1534 | state.build_res[builder] = { 1535 | 'res': bool(res) if res is not None else None, 1536 | 'url': url, 1537 | } 1538 | 1539 | db_query(db, 'SELECT repo, num, mergeable FROM mergeable') 1540 | for repo_label, num, mergeable in db.fetchall(): 1541 | try: 1542 | state = states[repo_label][num] 1543 | except KeyError: 1544 | db_query( 1545 | db, 1546 | 'DELETE FROM mergeable WHERE repo = ? AND num = ?', 1547 | [repo_label, num]) 1548 | continue 1549 | 1550 | state.mergeable = bool(mergeable) if mergeable is not None else None 1551 | 1552 | db_query(db, 'SELECT repo FROM pull GROUP BY repo') 1553 | for repo_label, in db.fetchall(): 1554 | if repo_label not in repos: 1555 | db_query(db, 'DELETE FROM pull WHERE repo = ?', [repo_label]) 1556 | 1557 | queue_handler_lock = Lock() 1558 | 1559 | def queue_handler(): 1560 | with queue_handler_lock: 1561 | return process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, git_cfg) # noqa 1562 | 1563 | os.environ['GIT_SSH'] = os.path.join(os.path.dirname(__file__), 'git_helper.py') # noqa 1564 | os.environ['GIT_EDITOR'] = 'cat' 1565 | 1566 | from . import server 1567 | Thread( 1568 | target=server.start, 1569 | args=[ 1570 | cfg, 1571 | states, 1572 | queue_handler, 1573 | repo_cfgs, 1574 | repos, 1575 | logger, 1576 | buildbot_slots, 1577 | my_username, 1578 | db, 1579 | repo_labels, 1580 | mergeable_que, 1581 | gh, 1582 | ]).start() 1583 | 1584 | Thread(target=fetch_mergeability, args=[mergeable_que]).start() 1585 | 1586 | queue_handler() 1587 | 1588 | 1589 | if __name__ == '__main__': 1590 | main() 1591 | --------------------------------------------------------------------------------