├── homu ├── tests │ ├── __init__.py │ ├── test_pr_body.py │ └── test_parse_issue_comment.py ├── git_helper.py ├── html │ ├── 404.html │ ├── retry_log.html │ ├── build_res.html │ ├── index.html │ └── queue.html ├── auth.py ├── utils.py ├── comments.py ├── parse_issue_comment.py ├── server.py └── main.py ├── .gitattributes ├── setup.cfg ├── .gitignore ├── requirements.txt ├── setup.py ├── LICENSE ├── Dockerfile ├── .github └── workflows │ └── ci.yml ├── cfg.production.toml ├── README.md └── cfg.sample.toml /homu/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.min.js binary 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /homu/__pycache__/ 2 | /.venv/ 3 | /cfg.toml 4 | /cfg.json 5 | /homu.egg-info/ 6 | /.eggs/ 7 | /main.db 8 | /cache 9 | *.pyc 10 | -------------------------------------------------------------------------------- /homu/git_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import os 5 | 6 | SSH_KEY_FILE = os.path.join(os.path.dirname(__file__), '../cache/key') 7 | 8 | 9 | def main(): 10 | args = ['ssh', '-i', SSH_KEY_FILE, '-S', 'none'] + sys.argv[1:] 11 | os.execvp('ssh', args) 12 | 13 | 14 | if __name__ == '__main__': 15 | main() 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | bottle==0.12.25 2 | certifi==2024.2.2 3 | charset-normalizer==3.3.2 4 | github3.py==0.9.6 5 | idna==3.6 6 | iniconfig==2.1.0 7 | Jinja2==3.1.3 8 | MarkupSafe==2.1.5 9 | pip==20.0.2 10 | pluggy==1.5.0 11 | requests==2.31.0 12 | retrying==1.3.4 13 | setuptools==45.2.0 14 | six==1.16.0 15 | toml==0.10.2 16 | uritemplate==4.1.1 17 | uritemplate.py==3.0.2 18 | urllib3==2.2.1 19 | waitress==3.0.0 20 | wheel==0.34.2 21 | -------------------------------------------------------------------------------- /homu/html/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Page not found 6 | 11 | 12 | 13 |

Page not found

14 |

Go back to the index

15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /homu/tests/test_pr_body.py: -------------------------------------------------------------------------------- 1 | from homu.main import ( 2 | suppress_ignore_block, 3 | suppress_pings, 4 | IGNORE_BLOCK_START, 5 | IGNORE_BLOCK_END, 6 | ) 7 | 8 | 9 | def test_suppress_pings_in_PR_body(): 10 | body = ( 11 | "r? @matklad\n" # should escape 12 | "@bors r+\n" # shouldn't 13 | "mail@example.com" # shouldn't 14 | ) 15 | 16 | expect = ( 17 | "r? `@matklad`\n" 18 | "`@bors` r+\n" 19 | "mail@example.com" 20 | ) 21 | 22 | assert suppress_pings(body) == expect 23 | 24 | 25 | def test_suppress_ignore_block_in_PR_body(): 26 | body = ( 27 | "Rollup merge\n" 28 | "{}\n" 29 | "[Create a similar rollup](https://fake.xyz/?prs=1,2,3)\n" 30 | "{}" 31 | ) 32 | 33 | body = body.format(IGNORE_BLOCK_START, IGNORE_BLOCK_END) 34 | 35 | expect = "Rollup merge\n" 36 | 37 | assert suppress_ignore_block(body) == expect 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='homu', 5 | version='0.3.0', 6 | author='Barosl Lee', 7 | url='https://github.com/barosl/homu', 8 | test_suite='homu.tests', 9 | description=('A bot that integrates with GitHub ' 10 | 'and your favorite continuous integration service'), 11 | 12 | packages=['homu'], 13 | install_requires=[ 14 | 'github3.py==0.9.6', 15 | 'toml', 16 | 'Jinja2', 17 | 'requests', 18 | 'bottle', 19 | 'waitress', 20 | 'retrying', 21 | ], 22 | setup_requires=[ 23 | 'pytest-runner<8', 24 | ], 25 | tests_require=[ 26 | 'pytest<8', 27 | 'typing_extensions<4.14' 28 | ], 29 | package_data={ 30 | 'homu': [ 31 | 'html/*.html', 32 | 'assets/*', 33 | ], 34 | }, 35 | entry_points={ 36 | 'console_scripts': [ 37 | 'homu=homu.main:main', 38 | ], 39 | }, 40 | zip_safe=False, 41 | ) 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Barosl Lee 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:focal 2 | # We need an older Ubuntu as github3 depends on < Python 3.10 to avoid errors 3 | 4 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 5 | python3-pip \ 6 | git \ 7 | ssh 8 | 9 | COPY setup.py cfg.production.toml /src/ 10 | COPY homu/ /src/homu/ 11 | COPY requirements.txt /src/ 12 | 13 | # Pre-install dependencies from a lockfile 14 | RUN pip3 install -r /src/requirements.txt 15 | 16 | # Homu needs to be installed in "editable mode" (-e): when pip installs an 17 | # application it resets the permissions of all source files to 644, but 18 | # homu/git_helper.py needs to be executable (755). Installing in editable mode 19 | # works around the issue since pip just symlinks the package to the source 20 | # directory. 21 | RUN pip3 install -e /src/ 22 | 23 | # Ensure the host SSH key for github.com is trusted by the container. If this 24 | # is not run, homu will fail to authenticate SSH connections with GitHub. 25 | RUN mkdir /root/.ssh && \ 26 | ssh-keyscan github.com >> /root/.ssh/known_hosts 27 | 28 | # Allow logs to show up timely on CloudWatch. 29 | ENV PYTHONUNBUFFERED=1 30 | 31 | CMD ["homu", "--verbose", "--config", "/src/cfg.production.toml"] 32 | -------------------------------------------------------------------------------- /homu/html/retry_log.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu retry log {{repo_label}} 6 | 17 | 18 | 19 |

Homu retry log - {{repo_label}}

20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | {% for log in logs %} 32 | 33 | 34 | 35 | 36 | 37 | {% endfor %} 38 | 39 |
Time (UTC)PRMessage
{{log.time}}{{log.num}}
{{log.msg}}
40 | 41 | 42 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | name: CI/CD 4 | on: 5 | push: 6 | branches: 7 | - master 8 | pull_request: {} 9 | 10 | env: 11 | PYTHON_VERSION: 3.8 12 | 13 | jobs: 14 | cicd: 15 | name: Test and deploy 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Clone the source code 19 | uses: actions/checkout@v2 20 | 21 | - name: Setup Python ${{ env.PYTHON_VERSION }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ env.PYTHON_VERSION }} 25 | 26 | - name: Install flake8 27 | run: pip install flake8 28 | 29 | - name: Ensure the code passes lints 30 | run: flake8 homu/ 31 | 32 | - name: Preinstall pinned Python dependencies 33 | run: pip install -r requirements.txt 34 | 35 | - name: Install homu on the builder 36 | run: pip install -e . 37 | 38 | - name: Run the test suite 39 | run: python3 setup.py test 40 | 41 | - name: Build the Docker image 42 | run: docker build -t homu . 43 | 44 | - name: Upload the Docker image to AWS ECR 45 | uses: rust-lang/simpleinfra/github-actions/upload-docker-image@master 46 | with: 47 | image: homu 48 | repository: bors 49 | region: us-west-1 50 | redeploy_ecs_cluster: rust-ecs-prod 51 | redeploy_ecs_service: bors 52 | aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} 53 | aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 54 | if: github.event_name == 'push' && github.repository == 'rust-lang/homu' && github.ref == 'refs/heads/master' 55 | -------------------------------------------------------------------------------- /homu/html/build_res.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu build result {{repo_label}}#{{pull}} 6 | 31 | 32 | 33 |

Homu build results - {{repo_label}}#{{pull}}

34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | {% for builder in builders %} 47 | 48 | 49 | 50 | 57 | 58 | {% endfor %} 59 | 60 |
Sort keyBuilderStatus
{{loop.index}}{{builder.name}} 51 | {%- if builder.url -%} 52 | {{builder.result}} 53 | {%- else -%} 54 | {{ builder.result }} 55 | {%- endif -%} 56 |
61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /cfg.production.toml: -------------------------------------------------------------------------------- 1 | max_priority = 9001 2 | 3 | [db] 4 | file = '/efs/main.db' 5 | 6 | [github] 7 | access_token = "${GITHUB_TOKEN}" 8 | app_client_id = "${GITHUB_CLIENT_ID}" 9 | app_client_secret = "${GITHUB_CLIENT_SECRET}" 10 | 11 | [git] 12 | local_git = true 13 | ssh_key = """ 14 | ${HOMU_SSH_KEY} 15 | """ 16 | 17 | [web] 18 | host = '0.0.0.0' 19 | port = 80 20 | 21 | base_url = "https://bors.rust-lang.org" 22 | canonical_url = "https://bors.rust-lang.org" 23 | remove_path_prefixes = ["homu"] 24 | 25 | #announcement = "Hello world!" 26 | 27 | ########## 28 | # Rust # 29 | ########## 30 | 31 | [repo.rust] 32 | owner = "rust-lang" 33 | name = "rust" 34 | timeout = 21600 # 6 hours 35 | 36 | # Permissions managed through rust-lang/team 37 | rust_team = true 38 | reviewers = [] 39 | try_users = [] 40 | 41 | [repo.rust.github] 42 | secret = "${HOMU_WEBHOOK_SECRET_RUST}" 43 | [repo.rust.checks.actions] 44 | name = "bors build finished" 45 | 46 | # Automatic relabeling 47 | [repo.rust.labels.approved] # after homu received `r+` 48 | remove = ['S-blocked', 'S-waiting-on-author', 'S-waiting-on-bors', 'S-waiting-on-crater', 'S-waiting-on-review', 'S-waiting-on-team'] 49 | add = ['S-waiting-on-bors'] 50 | 51 | [repo.rust.labels.rejected] # after homu received `r-` 52 | remove = ['S-blocked', 'S-waiting-on-author', 'S-waiting-on-bors', 'S-waiting-on-crater', 'S-waiting-on-review', 'S-waiting-on-team'] 53 | add = ['S-waiting-on-author'] 54 | 55 | [repo.rust.labels.failed] # test failed (maybe spurious, so fall back to -on-review) 56 | remove = ['S-blocked', 'S-waiting-on-author', 'S-waiting-on-bors', 'S-waiting-on-crater', 'S-waiting-on-review', 'S-waiting-on-team'] 57 | add = ['S-waiting-on-review'] 58 | 59 | [repo.rust.labels.timed_out] # test timed out after 4 hours (almost always spurious, let reviewer retry) 60 | remove = ['S-blocked', 'S-waiting-on-author', 'S-waiting-on-bors', 'S-waiting-on-crater', 'S-waiting-on-review', 'S-waiting-on-team'] 61 | add = ['S-waiting-on-review'] 62 | 63 | [repo.rust.labels.try_failed] # try-build failed (almost always legit, tell author to fix the PR) 64 | remove = ['S-waiting-on-review', 'S-waiting-on-crater'] 65 | add = ['S-waiting-on-author'] 66 | 67 | [repo.rust.labels.pushed] # user pushed a commit after `r+`/`try` 68 | remove = ['S-waiting-on-bors', 'S-waiting-on-author'] 69 | add = ['S-waiting-on-review'] 70 | unless = ['S-blocked', 'S-waiting-on-crater', 'S-waiting-on-team'] 71 | 72 | [repo.rust.labels.conflict] # a merge conflict is detected (tell author to rebase) 73 | remove = ['S-waiting-on-bors'] 74 | add = ['S-waiting-on-author'] 75 | unless = ['S-blocked', 'S-waiting-on-crater', 'S-waiting-on-team', 'S-waiting-on-review'] 76 | 77 | [repo.rust.labels.succeed] 78 | add = ['merged-by-bors'] 79 | 80 | [repo.rust.labels.rollup_made] 81 | add = ['rollup'] 82 | -------------------------------------------------------------------------------- /homu/auth.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | RUST_TEAM_BASE = "https://team-api.infra.rust-lang.org/v1/" 4 | RETRIES = 5 5 | 6 | 7 | def fetch_rust_team(repo_label, level): 8 | repo = repo_label.replace('-', '_') 9 | url = RUST_TEAM_BASE + "permissions/bors." + repo + "." + level + ".json" 10 | for retry in range(RETRIES): 11 | try: 12 | resp = requests.get(url) 13 | resp.raise_for_status() 14 | return resp.json()["github_ids"] 15 | except requests.exceptions.RequestException as e: 16 | msg = "error while fetching " + url 17 | msg += " (try " + str(retry) + "): " + str(e) 18 | print(msg) 19 | continue 20 | return [] 21 | 22 | 23 | def verify_level(username, user_id, repo_label, repo_cfg, state, toml_keys, 24 | rust_team_level): 25 | authorized = False 26 | if repo_cfg.get('auth_collaborators', False): 27 | authorized = state.get_repo().is_collaborator(username) 28 | if repo_cfg.get('rust_team', False): 29 | authorized = user_id in fetch_rust_team(repo_label, rust_team_level) 30 | if not authorized: 31 | authorized = username.lower() == state.delegate.lower() 32 | for toml_key in toml_keys: 33 | if not authorized: 34 | authorized = username in repo_cfg.get(toml_key, []) 35 | return authorized 36 | 37 | 38 | def verify(username, user_id, repo_label, repo_cfg, state, auth, realtime, 39 | my_username): 40 | # The import is inside the function to prevent circular imports: main.py 41 | # requires auth.py and auth.py requires main.py 42 | from .main import AuthState 43 | 44 | # In some cases (e.g. non-fully-qualified r+) we recursively talk to 45 | # ourself via a hidden markdown comment in the message. This is so that 46 | # when re-synchronizing after shutdown we can parse these comments and 47 | # still know the SHA for the approval. 48 | # 49 | # So comments from self should always be allowed 50 | if username == my_username: 51 | return True 52 | 53 | authorized = False 54 | if auth == AuthState.REVIEWER: 55 | authorized = verify_level( 56 | username, user_id, repo_label, repo_cfg, state, ['reviewers'], 57 | 'review', 58 | ) 59 | elif auth == AuthState.TRY: 60 | authorized = verify_level( 61 | username, user_id, repo_label, repo_cfg, state, 62 | ['reviewers', 'try_users'], 'try', 63 | ) 64 | 65 | if authorized: 66 | return True 67 | else: 68 | if realtime: 69 | reply = '@{}: :key: Insufficient privileges: '.format(username) 70 | if auth == AuthState.REVIEWER: 71 | if repo_cfg.get('auth_collaborators', False): 72 | reply += 'Collaborator required' 73 | else: 74 | reply += 'Not in reviewers' 75 | elif auth == AuthState.TRY: 76 | reply += 'not in try users' 77 | state.add_comment(reply) 78 | return False 79 | -------------------------------------------------------------------------------- /homu/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import github3 3 | import logging 4 | import subprocess 5 | import sys 6 | import traceback 7 | import requests 8 | import time 9 | 10 | 11 | def github_set_ref(repo, ref, sha, *, force=False, auto_create=True, retry=1): 12 | url = repo._build_url('git', 'refs', ref, base_url=repo._api) 13 | data = {'sha': sha, 'force': force} 14 | 15 | try: 16 | js = repo._json(repo._patch(url, data=json.dumps(data)), 200) 17 | except github3.models.GitHubError as e: 18 | if e.code == 422 and auto_create: 19 | try: 20 | return repo.create_ref('refs/' + ref, sha) 21 | except github3.models.GitHubError: 22 | raise e 23 | elif e.code == 422 and retry > 0: 24 | time.sleep(5) 25 | return github_set_ref(repo, 26 | ref, 27 | sha, 28 | force=force, 29 | auto_create=auto_create, 30 | retry=retry - 1) 31 | else: 32 | raise 33 | 34 | return github3.git.Reference(js, repo) if js else None 35 | 36 | 37 | class Status(github3.repos.status.Status): 38 | def __init__(self, info): 39 | super(Status, self).__init__(info) 40 | 41 | self.context = info.get('context') 42 | 43 | 44 | def github_iter_statuses(repo, sha): 45 | url = repo._build_url('statuses', sha, base_url=repo._api) 46 | return repo._iter(-1, url, Status) 47 | 48 | 49 | def github_create_status(repo, sha, state, target_url='', description='', *, 50 | context=''): 51 | data = {'state': state, 'target_url': target_url, 52 | 'description': description, 'context': context} 53 | url = repo._build_url('statuses', sha, base_url=repo._api) 54 | js = repo._json(repo._post(url, data=data), 201) 55 | return Status(js) if js else None 56 | 57 | 58 | def remove_url_keys_from_json(json): 59 | if isinstance(json, dict): 60 | return {key: remove_url_keys_from_json(value) 61 | for key, value in json.items() 62 | if not key.endswith('url')} 63 | elif isinstance(json, list): 64 | return [remove_url_keys_from_json(value) for value in json] 65 | else: 66 | return json 67 | 68 | 69 | def lazy_debug(logger, f): 70 | if logger.isEnabledFor(logging.DEBUG): 71 | logger.debug(f()) 72 | 73 | 74 | def logged_call(args): 75 | try: 76 | subprocess.check_call(args, stdout=subprocess.DEVNULL, stderr=None) 77 | except subprocess.CalledProcessError: 78 | print('* Failed to execute command: {}'.format(args)) 79 | raise 80 | 81 | 82 | def silent_call(args): 83 | return subprocess.call( 84 | args, 85 | stdout=subprocess.DEVNULL, 86 | stderr=subprocess.DEVNULL, 87 | ) 88 | 89 | 90 | def retry_until(inner, fail, state): 91 | err = None 92 | exc_info = None 93 | 94 | for i in range(3, 0, -1): 95 | try: 96 | inner() 97 | except (github3.models.GitHubError, requests.exceptions.RequestException) as e: # noqa 98 | print('* Intermittent GitHub error: {}'.format(e), file=sys.stderr) 99 | 100 | err = e 101 | exc_info = sys.exc_info() 102 | 103 | if i != 1: 104 | time.sleep(1) 105 | else: 106 | err = None 107 | break 108 | 109 | if err: 110 | print('* GitHub failure in {}'.format(state), file=sys.stderr) 111 | traceback.print_exception(*exc_info) 112 | 113 | fail(err) 114 | -------------------------------------------------------------------------------- /homu/html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu 6 | 29 | 30 | 31 | {% if announcement != None %} 32 |
{{ announcement | safe }}
33 | {% endif %} 34 | 35 |
36 | 37 |

Homu

38 | 39 |

Repositories

40 | 41 | 46 | 47 |
48 | 49 |

Homu Cheatsheet

50 | 51 |

Commands

52 | 53 |

54 | Here's a quick reference for the commands Homu accepts. Commands must be posted as 55 | comments on the PR they refer to. Comments may include multiple commands. Homu will 56 | only listen to official reviewers that it is configured to listen to. A comment 57 | must mention the GitHub account Homu is configured to use. (e.g. for the Rust project this is @bors) 58 | Note that Homu will only recognize comments in open PRs. 59 |

60 | 61 | 79 | 80 |

Examples

81 | 82 | 89 | 90 |

Customizing the Queue's Contents

91 | 92 |

93 | Homu provides a few simple ways to customize the queue's contents to fit your needs: 94 |

95 | 96 | 102 | 103 |
104 | 105 | 106 | -------------------------------------------------------------------------------- /homu/comments.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class Comment: 5 | def __init__(self, **args): 6 | if len(args) != len(self.params): 7 | raise KeyError("different number of params") 8 | for key, value in args.items(): 9 | if key in self.params: 10 | setattr(self, key, value) 11 | else: 12 | raise KeyError("unknown attribute: %s" % key) 13 | 14 | def jsonify(self): 15 | out = {"type": self.__class__.__name__} 16 | for param in self.params: 17 | out[param] = getattr(self, param) 18 | return json.dumps(out, separators=(',', ':')) 19 | 20 | 21 | class Approved(Comment): 22 | def __init__(self, bot=None, **args): 23 | # Because homu needs to leave a comment for itself to kick off a build, 24 | # we need to know the correct botname to use. However, we don't want to 25 | # save that botname in our state JSON. So we need a custom constructor 26 | # to grab the botname and delegate the rest of the keyword args to the 27 | # Comment constructor. 28 | super().__init__(**args) 29 | self.bot = bot 30 | 31 | params = ["sha", "approver", "queue"] 32 | 33 | def render(self): 34 | # The comment here is required because Homu wants a full, unambiguous, 35 | # pinned commit hash to kick off the build, and this note-to-self is 36 | # how it gets it. This is to safeguard against situations where Homu 37 | # reloads and another commit has been pushed since the approval. 38 | message = ":pushpin: Commit {sha} has been " + \ 39 | "approved by `{approver}`\n\n" + \ 40 | "It is now in the [queue]({queue}) for this repository.\n\n" + \ 41 | "" 42 | return message.format( 43 | sha=self.sha, 44 | approver=self.approver, 45 | bot=self.bot, 46 | queue=self.queue 47 | ) 48 | 49 | 50 | class ApprovalIgnoredWip(Comment): 51 | def __init__(self, wip_keyword=None, **args): 52 | # We want to use the wip keyword in the message, but not in the json 53 | # blob. 54 | super().__init__(**args) 55 | self.wip_keyword = wip_keyword 56 | 57 | params = ["sha"] 58 | 59 | def render(self): 60 | message = ':clipboard:' + \ 61 | ' Looks like this PR is still in progress,' + \ 62 | ' ignoring approval.\n\n' + \ 63 | 'Hint: Remove **{wip_keyword}** from this PR\'s title when' + \ 64 | ' it is ready for review.' 65 | return message.format(wip_keyword=self.wip_keyword) 66 | 67 | 68 | class Delegated(Comment): 69 | def __init__(self, bot=None, **args): 70 | # Because homu needs to leave a comment for the delegated person, 71 | # we need to know the correct botname to use. However, we don't want to 72 | # save that botname in our state JSON. So we need a custom constructor 73 | # to grab the botname and delegate the rest of the keyword args to the 74 | # Comment constructor. 75 | super().__init__(**args) 76 | self.bot = bot 77 | 78 | params = ["delegator", "delegate"] 79 | 80 | def render(self): 81 | message = \ 82 | ':v: @{delegate}, you can now approve this pull request!\n\n' + \ 83 | 'If @{delegator} told you to "`r=me`" after making some ' + \ 84 | 'further change, please make that change, then do ' + \ 85 | '`@{bot} r=@{delegator}`' 86 | return message.format( 87 | delegate=self.delegate, 88 | bot=self.bot, 89 | delegator=self.delegator 90 | ) 91 | 92 | 93 | class BuildStarted(Comment): 94 | params = ["head_sha", "merge_sha"] 95 | 96 | def render(self): 97 | return ":hourglass: Testing commit %s with merge %s..." % ( 98 | self.head_sha, self.merge_sha, 99 | ) 100 | 101 | 102 | class TryBuildStarted(Comment): 103 | params = ["head_sha", "merge_sha"] 104 | 105 | def render(self): 106 | return ":hourglass: Trying commit %s with merge %s..." % ( 107 | self.head_sha, self.merge_sha, 108 | ) 109 | 110 | 111 | class BuildCompleted(Comment): 112 | params = ["approved_by", "base_ref", "builders", "merge_sha"] 113 | 114 | def render(self): 115 | urls = ", ".join( 116 | "[%s](%s)" % kv for kv in sorted(self.builders.items()) 117 | ) 118 | return ( 119 | ":sunny: Test successful - %s\n" 120 | "Approved by: %s\n" 121 | "Pushing %s to %s..." 122 | % ( 123 | urls, self.approved_by, self.merge_sha, self.base_ref, 124 | ) 125 | ) 126 | 127 | 128 | class TryBuildCompleted(Comment): 129 | params = ["builders", "merge_sha"] 130 | 131 | def render(self): 132 | urls = ", ".join( 133 | "[%s](%s)" % kv for kv in sorted(self.builders.items()) 134 | ) 135 | return ":sunny: Try build successful - %s\nBuild commit: %s (`%s`)" % ( 136 | urls, self.merge_sha, self.merge_sha, 137 | ) 138 | 139 | 140 | class BuildFailed(Comment): 141 | params = ["builder_url", "builder_name"] 142 | 143 | def render(self): 144 | return ":broken_heart: Test failed - [%s](%s)" % ( 145 | self.builder_name, self.builder_url 146 | ) 147 | 148 | 149 | class TryBuildFailed(Comment): 150 | params = ["builder_url", "builder_name"] 151 | 152 | def render(self): 153 | return ":broken_heart: Test failed - [%s](%s)" % ( 154 | self.builder_name, self.builder_url 155 | ) 156 | 157 | 158 | class TimedOut(Comment): 159 | params = [] 160 | 161 | def render(self): 162 | return ":boom: Test timed out" 163 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Homu 2 | 3 | [![Hommando]][Akemi Homura] 4 | 5 | Homu is a bot that integrates with GitHub and your favorite continuous 6 | integration service such as [Travis CI], [Appveyor] or [Buildbot]. 7 | 8 | [Hommando]: https://i.imgur.com/j0jNvHF.png 9 | [Akemi Homura]: https://wiki.puella-magi.net/Homura_Akemi 10 | [Buildbot]: http://buildbot.net/ 11 | [Travis CI]: https://travis-ci.org/ 12 | [Appveyor]: https://www.appveyor.com/ 13 | 14 | ## Why is it needed? 15 | 16 | Let's take Travis CI as an example. If you send a pull request to a repository, 17 | Travis CI instantly shows you the test result, which is great. However, after 18 | several other pull requests are merged into the default branch, your pull 19 | request can *still* break things after being merged into the default branch. The 20 | traditional continuous integration solutions don't protect you from this. 21 | 22 | In fact, that's why they provide the build status badges. If anything pushed to 23 | the default branch is completely free from any breakage, those badges will **not** be 24 | necessary, as they will always be green. The badges themselves prove that there 25 | can still be some breakages, even when continuous integration services are used. 26 | 27 | To solve this problem, the test procedure should be executed *just before the 28 | merge*, not just after the pull request is received. You can manually click the 29 | "restart build" button each time before you merge a pull request, but Homu can 30 | automate this process. It listens to the pull request comments, waiting for an 31 | approval comment from one of the configured reviewers. When the pull request is 32 | approved, Homu tests it using your favorite continuous integration service, and 33 | only when it passes all the tests, it is merged into the default branch. 34 | 35 | Note that Homu is **not** a replacement of Travis CI, Buildbot or Appveyor. It 36 | works on top of them. Homu itself doesn't have the ability to test pull 37 | requests. 38 | 39 | ## Influences of bors 40 | 41 | Homu is largely inspired by [bors]. The concept of "tests should be done just 42 | before the merge" came from bors. However, there are also some differences: 43 | 44 | 1. Stateful: Unlike bors, which intends to be stateless, Homu is stateful. It 45 | means that Homu does not need to retrieve all the information again and again 46 | from GitHub at every run. This is essential because of GitHub's rate 47 | limiting. Once it downloads the initial state, the following changes are 48 | delivered with the [Webhooks] API. 49 | 2. Pushing over polling: Homu prefers pushing wherever possible. The pull 50 | requests from GitHub are retrieved using Webhooks, as stated above. The test 51 | results from Buildbot are pushed back to Homu with the [HttpStatusPush] 52 | feature. This approach improves the overall performance and the response 53 | time, because the bot is informed about the status changes immediately. 54 | 55 | And also, Homu has more features, such as `rollup`, `try`, and the Travis CI & 56 | Appveyor support. 57 | 58 | [bors]: https://github.com/graydon/bors 59 | [Webhooks]: https://developer.github.com/webhooks/ 60 | [HttpStatusPush]: http://docs.buildbot.net/current/manual/cfg-statustargets.html#httpstatuspush 61 | 62 | ## Usage 63 | 64 | ### How to install 65 | 66 | ```sh 67 | $ sudo apt-get install python3-venv python3-wheel 68 | $ python3 -m venv .venv 69 | $ . .venv/bin/activate 70 | $ pip install -U pip 71 | $ git clone https://github.com/rust-lang/homu.git 72 | $ pip install -e homu 73 | ``` 74 | 75 | ### How to configure 76 | 77 | In the following instructions, `HOST` refers to the hostname (or IP address) 78 | where you are running your custom homu instance. `PORT` is the port the service 79 | is listening to and is configured in `web.port` in `cfg.toml`. `NAME` refers to 80 | the name of the repository you are configuring homu for. 81 | 82 | 1. Copy `cfg.sample.toml` to `cfg.toml`. You'll need to edit this file to set up 83 | your configuration. The following steps explain where you can find important 84 | config values. 85 | 86 | 2. Create a GitHub account that will be used by Homu. You can also use an 87 | existing account. In the [developer settings][settings], go to "OAuth 88 | Apps" and create a new application: 89 | - Make note of the "Client ID" and "Client Secret"; you will need to put them in 90 | your `cfg.toml`. 91 | - The OAuth Callback URL should be `http://HOST:PORT/callback`. 92 | - The homepage URL isn't necessary; you could set `http://HOST:PORT/`. 93 | 94 | 3. Go back to the developer settings of the GitHub account you created/used in the 95 | previous step. Go to "Personal access tokens". Click "Generate new token" and 96 | choose the "repo" and "user" scopes. Put the token value in your `cfg.toml`. 97 | 98 | 4. Add your new GitHub account as a Collaborator to the GitHub repo you are 99 | setting up homu for. This can be done in repo (NOT user) "Settings", then 100 | "Collaborators". Enable "Write" access. 101 | 102 | 4.1. Make sure you login as the new GitHub account and that you **accept 103 | the collaborator invitation** you just sent! 104 | 105 | 5. Add a Webhook to your repository. This is done under repo (NOT user) 106 | "Settings", then "Webhooks". Click "Add webhook", then set: 107 | - Payload URL: `http://HOST:PORT/github` 108 | - Content type: `application/json` 109 | - Secret: The same as `repo.NAME.github.secret` in `cfg.toml` 110 | - Events: click "Let me select individual events", then pick 111 | `Issue comments`, `Pull requests`, `Pushes`, `Statuses`, `Check runs` 112 | 113 | 6. Add a Webhook to your continuous integration service, if necessary. You don't 114 | need this if using Travis/Appveyor. 115 | - Buildbot 116 | 117 | Insert the following code to the `master.cfg` file: 118 | 119 | ```python 120 | from buildbot.status.status_push import HttpStatusPush 121 | 122 | c['status'].append(HttpStatusPush( 123 | serverUrl='http://HOST:PORT/buildbot', 124 | extra_post_params={'secret': 'repo.NAME.buildbot.secret in cfg.toml'}, 125 | )) 126 | ``` 127 | 128 | 7. Go through the rest of your `cfg.toml` and uncomment (and change, if needed) 129 | parts of the config you'll need. 130 | 131 | [settings]: https://github.com/settings/apps 132 | [travis]: https://travis-ci.org/profile/info 133 | 134 | ### How to run 135 | 136 | ```sh 137 | $ . .venv/bin/activate 138 | $ homu 139 | ``` 140 | -------------------------------------------------------------------------------- /cfg.sample.toml: -------------------------------------------------------------------------------- 1 | # The configuration file supports variable interpolation. In any string field, 2 | # ${VARIABLE_NAME} will be replaced with the value of the VARIABLE_NAME 3 | # environment variable. 4 | 5 | # Priority values above max_priority will be refused. 6 | max_priority = 9001 7 | 8 | # How long to keep the retry log 9 | # Should be a negative interval of time recognized by SQLite3. 10 | retry_log_expire = '-42 days' 11 | 12 | [github] 13 | 14 | # Information for securely interacting with GitHub. These are found/generated 15 | # under . 16 | 17 | # A GitHub personal access token. 18 | access_token = "" 19 | 20 | # A GitHub oauth application for this instance of homu: 21 | app_client_id = "" 22 | app_client_secret = "" 23 | 24 | 25 | [git] 26 | # Use the local Git command. Required to use some advanced features. It also 27 | # speeds up Travis by reducing temporary commits. 28 | #local_git = false 29 | 30 | # Directory storing the local clones of the git repositories. If this is on an 31 | # ephemeral file system, there will be a delay to start new builds after a 32 | # restart while homu clones the repository. 33 | # cache_dir = "cache" 34 | 35 | # SSH private key. Needed only when the local Git command is used. 36 | #ssh_key = """ 37 | #""" 38 | 39 | # By default, Homu extracts the name+email from the Github account it will be 40 | # using. However, you may want to use a private email for the account, and 41 | # associate the commits with a public email address. 42 | #user = "Some Cool Project Bot" 43 | #email = "coolprojectbot-devel@example.com" 44 | 45 | [web] 46 | 47 | # The port homu listens on. 48 | port = 54856 49 | 50 | # Synchronize all open PRs on startup. "Synchronize" means fetch the state of 51 | # all open PRs. 52 | sync_on_start = true 53 | 54 | # The base url used for links pointing to this homu instance. 55 | # If base_url is not present, links will use canonical_url as a fallback. 56 | # If neither base_url nor canonical_url are present, no links to this homu 57 | # instance will be generated. 58 | #base_url = "https://bors.example.com" 59 | 60 | # The canonical URL of this homu instance. If a user reaches the instance 61 | # through a different path they will be redirected. If this is not present in 62 | # the configuration homu will still work, but no redirect will be performed. 63 | #canonical_url = "https://bors.example.com" 64 | 65 | # List of path prefixes to remove from the URL. This is useful if the homu 66 | # instance was moved from a subdirectory of a domain to the top level. 67 | #remove_path_prefixes = ["homu"] 68 | 69 | # Message to display on top of the web interface. It's possible to use 70 | # arbitrary HTML tags in the message. 71 | #announcement = "Homu will be offline tomorrow for maintenance." 72 | 73 | # Custom hooks can be added as well. 74 | # Homu will ping the given endpoint with POSTdata of the form: 75 | # {'body': 'comment body', 'extra_data': 'extra data', 'pull': pull req number} 76 | # The extra data is the text specified in `@homu hookname=text` 77 | # 78 | # [hooks.hookname] 79 | # trigger = "hookname" # will be triggered by @homu hookname or @homu hookname=text 80 | # endpoint = "http://path/to/endpoint" 81 | # access = "try" # access level required 82 | # has_response = true # Should the response be posted back to github? Only allowed if realtime=true 83 | # realtime = true # Should it only run in realtime mode? If false, this will be replayed each time homu is started (probably not what you want) 84 | 85 | # An example configuration for repository (there can be many of these). NAME 86 | # refers to your repo name. 87 | [repo.NAME] 88 | 89 | # Which repo are we talking about? You can get these fields from your repo URL: 90 | # github.com// 91 | owner = "" 92 | name = "" 93 | 94 | # If this repo should be integrated with the permissions defined in 95 | # https://github.com/rust-lang/team uncomment the following line. 96 | # Note that the other ACLs will *also* apply. 97 | #rust_team = true 98 | 99 | # Who can approve PRs (r+ rights)? You can put GitHub usernames here. 100 | reviewers = [] 101 | # Alternatively, set this allow any github collaborator; 102 | # note that you can *also* specify reviewers above. 103 | # auth_collaborators = true 104 | 105 | # Who has 'try' rights? (try, retry, force, clean, prioritization). It's fine to 106 | # keep this empty. 107 | try_users = [] 108 | 109 | # Keep the commit history linear. Requires the local Git command. 110 | #linear = false 111 | 112 | # Auto-squash commits. Requires the local Git command. 113 | #autosquash = true 114 | 115 | # If the PR already has the same success statuses that we expect on the `auto` 116 | # branch, then push directly to branch if safe to do so. Requires the local Git 117 | # command. 118 | #status_based_exemption = false 119 | 120 | # Maximum test duration allowed for testing a PR in this repository. 121 | # Default to 10 hours. 122 | #timeout = 36000 123 | 124 | # Branch names. These settings are the defaults; it makes sense to leave these 125 | # as-is. 126 | #[repo.NAME.branch] 127 | # 128 | #auto = "auto" 129 | #try = "try" 130 | 131 | # test-on-fork allows you to run the CI builds for a project in a separate fork 132 | # instead of the main repository, while still approving PRs and merging the 133 | # commits in the main one. 134 | # 135 | # To enable test-on-fork you need to uncomment the section below and fill the 136 | # fork's owner and repository name. The fork MUST BE AN ACTUAL GITHUB FORK for 137 | # this feature to work. That means it will likely need to be in a separate 138 | # GitHub organization. 139 | # 140 | # This only works when `local_git = true`. 141 | # 142 | #[repo.NAME.test-on-fork] 143 | #owner = "" 144 | #name = "" 145 | 146 | [repo.NAME.github] 147 | # Arbitrary secret. You can generate one with: openssl rand -hex 20 148 | secret = "" 149 | 150 | # Remove and add GitHub labels when some event happened. 151 | # See servo/homu#141 for detail. 152 | # 153 | #[repo.NAME.labels.approved] # after homu received `r+` 154 | #[repo.NAME.labels.rejected] # after homu received `r-` 155 | #[repo.NAME.labels.conflict] # a merge conflict is detected 156 | #[repo.NAME.labels.succeed] # test successful 157 | #[repo.NAME.labels.failed] # test failed 158 | #[repo.NAME.labels.exempted] # test exempted 159 | #[repo.NAME.labels.timed_out] # test timed out (after 10 hours) 160 | #[repo.NAME.labels.interrupted] # test interrupted (buildbot only) 161 | #[repo.NAME.labels.try] # after homu received `try` 162 | #[repo.NAME.labels.try_succeed] # try-build successful 163 | #[repo.NAME.labels.try_failed] # try-build failed 164 | #[repo.NAME.labels.pushed] # user pushed a commit after `r+`/`try` 165 | #remove = ['list', 'of', 'labels', 'to', 'remove'] 166 | #add = ['list', 'of', 'labels', 'to', 'add'] 167 | #unless = [ 168 | # 'avoid', 'relabeling', 'if', 169 | # 'any', 'of', 'these', 'labels', 'are', 'present', 170 | #] 171 | 172 | # Travis integration. Don't forget to allow Travis to test the `auto` branch! 173 | [repo.NAME.checks.travis] 174 | # Name of the Checks API run. Don't touch this unless you really know what 175 | # you're doing. 176 | name = "Travis CI - Branch" 177 | 178 | # Appveyor integration. Don't forget to allow Appveyor to test the `auto` branch! 179 | #[repo.NAME.status.appveyor] 180 | # 181 | # String label set by status updates. Don't touch this unless you really know 182 | # what you're doing. 183 | #context = 'continuous-integration/appveyor/branch' 184 | 185 | # Generic GitHub Status API support. You don't need this if you're using the 186 | # above examples for Travis/Appveyor. 187 | #[repo.NAME.status.LABEL] 188 | # 189 | # String label set by status updates. 190 | #context = "" 191 | # 192 | # Equivalent context to look for on the PR itself if checking whether the 193 | # build should be exempted. If omitted, looks for the same context. This is 194 | # only used if status_based_exemption is true. 195 | #pr_context = "" 196 | 197 | # Generic GitHub Checks API support. You don't need this if you're using the 198 | # above examples for Travis/Appveyor. 199 | #[repo.NAME.checks.LABEL] 200 | # 201 | # String name of the Checks run. 202 | #name = "" 203 | # 204 | # String name of the Checks run used for try runs. 205 | # If the field is omitted the same name as the auto build will be used. 206 | #try_name = "" 207 | 208 | # Use buildbot for running tests 209 | #[repo.NAME.buildbot] 210 | # 211 | #url = "" 212 | #secret = "" 213 | # 214 | #builders = ["auto-linux", "auto-mac"] 215 | #try_builders = ["try-linux", "try-mac"] 216 | # 217 | #username = "" 218 | #password = "" 219 | 220 | # 221 | ## Boolean which indicates whether the builder is included in try builds (defaults to true) 222 | #try = false 223 | 224 | # The database homu uses 225 | [db] 226 | # SQLite file 227 | file = "main.db" 228 | -------------------------------------------------------------------------------- /homu/parse_issue_comment.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | import re 3 | 4 | WORDS_TO_ROLLUP = { 5 | 'rollup-': 0, 6 | 'rollup': 1, 7 | 'rollup=maybe': 0, 8 | 'rollup=never': -2, 9 | 'rollup=iffy': -1, 10 | 'rollup=always': 1, 11 | } 12 | 13 | 14 | class IssueCommentCommand: 15 | """ 16 | A command that has been parsed out of a GitHub issue comment. 17 | 18 | E.g., `@bors r+` => an issue command with action == 'approve' 19 | """ 20 | 21 | def __init__(self, action): 22 | self.action = action 23 | 24 | @classmethod 25 | def approve(cls, approver, commit): 26 | command = cls('approve') 27 | command.commit = commit 28 | command.actor = approver.lstrip('@') 29 | return command 30 | 31 | @classmethod 32 | def unapprove(cls): 33 | return cls('unapprove') 34 | 35 | @classmethod 36 | def prioritize(cls, priority): 37 | command = cls('prioritize') 38 | command.priority = priority 39 | return command 40 | 41 | @classmethod 42 | def delegate_author(cls): 43 | return cls('delegate-author') 44 | 45 | @classmethod 46 | def delegate(cls, delegate_to): 47 | command = cls('delegate') 48 | command.delegate_to = delegate_to.lstrip('@') 49 | return command 50 | 51 | @classmethod 52 | def undelegate(cls): 53 | return cls('undelegate') 54 | 55 | @classmethod 56 | def retry(cls): 57 | return cls('retry') 58 | 59 | @classmethod 60 | def try_(cls): 61 | return cls('try') 62 | 63 | @classmethod 64 | def untry(cls): 65 | return cls('untry') 66 | 67 | @classmethod 68 | def rollup(cls, rollup_value): 69 | command = cls('rollup') 70 | command.rollup_value = rollup_value 71 | return command 72 | 73 | @classmethod 74 | def squash(cls): 75 | return cls('squash') 76 | 77 | @classmethod 78 | def unsquash(cls): 79 | return cls('unsquash') 80 | 81 | @classmethod 82 | def force(cls): 83 | return cls('force') 84 | 85 | @classmethod 86 | def clean(cls): 87 | return cls('clean') 88 | 89 | @classmethod 90 | def ping(cls, ping_type='standard'): 91 | command = cls('ping') 92 | command.ping_type = ping_type 93 | return command 94 | 95 | @classmethod 96 | def treeclosed(cls, treeclosed_value): 97 | command = cls('treeclosed') 98 | command.treeclosed_value = treeclosed_value 99 | return command 100 | 101 | @classmethod 102 | def untreeclosed(cls): 103 | return cls('untreeclosed') 104 | 105 | @classmethod 106 | def hook(cls, hook_name, hook_extra=None): 107 | command = cls('hook') 108 | command.hook_name = hook_name 109 | command.hook_extra = hook_extra 110 | return command 111 | 112 | 113 | def is_sha(sha): 114 | """ 115 | Try to determine if the input is a git sha 116 | """ 117 | return re.match(r'^[0-9a-f]{4,}$', sha) 118 | 119 | 120 | def hook_with_extra_is_in_hooks(word, hooks): 121 | """ 122 | Determine if the word given is the name of a valid hook, with extra data 123 | hanging off of it (e.g., `validhookname=extradata`). 124 | 125 | hook_with_extra_is_in_hooks( 126 | 'validhookname=stuff', 127 | ['validhookname', 'other']) 128 | #=> True 129 | 130 | hook_with_extra_is_in_hooks( 131 | 'invalidhookname=stuff', 132 | ['validhookname', 'other']) 133 | #=> False 134 | 135 | hook_with_extra_is_in_hooks( 136 | 'validhookname', 137 | ['validhookname', 'other']) 138 | #=> False 139 | """ 140 | for hook in hooks: 141 | if word.startswith('{}='.format(hook)): 142 | return True 143 | 144 | return False 145 | 146 | 147 | def parse_issue_comment(username, body, sha, botname, hooks=[]): 148 | """ 149 | Parse an issue comment looking for commands that Homu should handle 150 | 151 | Parameters: 152 | username: the username of the user that created the issue comment. 153 | This is without the leading @ 154 | body: the full body of the comment (markdown) 155 | sha: the commit that the comment applies to 156 | botname: the name of bot. This is without the leading @. 157 | So if we should respond to `@bors {command}`, botname will be `bors` 158 | hooks: a list of strings that are valid hook names. 159 | E.g. `['hook1', 'hook2', 'hook3']` 160 | """ 161 | 162 | botname_regex = re.compile(r'^.*(?=@' + botname + ')') 163 | 164 | # All of the 'words' after and including the botname 165 | words = list(chain.from_iterable( 166 | re.findall(r'\S+', re.sub(botname_regex, '', x)) 167 | for x 168 | in body.splitlines() 169 | if '@' + botname in x and not x.lstrip().startswith('>'))) # noqa 170 | 171 | commands = [] 172 | 173 | if words[1:] == ["are", "you", "still", "there?"]: 174 | commands.append(IssueCommentCommand.ping('portal')) 175 | 176 | for i, word in enumerate(words): 177 | if word is None: 178 | # We already parsed the next word, and we set it to an empty string 179 | # to signify that we did. 180 | continue 181 | 182 | if word == '@' + botname: 183 | continue 184 | 185 | if word == '@' + botname + ':': 186 | continue 187 | 188 | if word == 'r+' or word.startswith('r='): 189 | approved_sha = sha 190 | 191 | if i + 1 < len(words) and is_sha(words[i + 1]): 192 | approved_sha = words[i + 1] 193 | words[i + 1] = None 194 | 195 | approver = word[len('r='):] if word.startswith('r=') else username 196 | 197 | # Ignore "r=me" 198 | if approver == 'me': 199 | continue 200 | 201 | commands.append( 202 | IssueCommentCommand.approve(approver, approved_sha)) 203 | 204 | elif word == 'r-': 205 | commands.append(IssueCommentCommand.unapprove()) 206 | 207 | elif word.startswith('p='): 208 | try: 209 | pvalue = int(word[len('p='):]) 210 | except ValueError: 211 | continue 212 | 213 | commands.append(IssueCommentCommand.prioritize(pvalue)) 214 | 215 | elif word.startswith('delegate='): 216 | delegate = word[len('delegate='):] 217 | commands.append(IssueCommentCommand.delegate(delegate)) 218 | 219 | elif word == 'delegate-': 220 | commands.append(IssueCommentCommand.undelegate()) 221 | 222 | elif word == 'delegate+': 223 | commands.append(IssueCommentCommand.delegate_author()) 224 | 225 | elif word == 'retry': 226 | commands.append(IssueCommentCommand.retry()) 227 | 228 | # elif word == 'try': 229 | # commands.append(IssueCommentCommand.try_()) 230 | 231 | # elif word == 'try-': 232 | # Try- is broken, prevent its usage. 233 | # commands.append(IssueCommentCommand.untry()) 234 | # pass 235 | 236 | elif word in WORDS_TO_ROLLUP: 237 | rollup_value = WORDS_TO_ROLLUP[word] 238 | commands.append(IssueCommentCommand.rollup(rollup_value)) 239 | 240 | elif word == 'squash': 241 | # Squash is broken, prevent its usage. 242 | # commands.append(IssueCommentCommand.squash()) 243 | pass 244 | 245 | elif word == 'squash-': 246 | # Squash is broken, prevent its usage. 247 | # commands.append(IssueCommentCommand.unsquash()) 248 | pass 249 | 250 | elif word == 'force': 251 | commands.append(IssueCommentCommand.force()) 252 | 253 | elif word == 'clean': 254 | commands.append(IssueCommentCommand.clean()) 255 | 256 | elif (word == 'hello?' or word == 'ping'): 257 | commands.append(IssueCommentCommand.ping()) 258 | 259 | elif word.startswith('treeclosed='): 260 | try: 261 | treeclosed = int(word[len('treeclosed='):]) 262 | commands.append(IssueCommentCommand.treeclosed(treeclosed)) 263 | except ValueError: 264 | pass 265 | 266 | elif word == 'treeclosed-': 267 | commands.append(IssueCommentCommand.untreeclosed()) 268 | 269 | elif word in hooks: 270 | commands.append(IssueCommentCommand.hook(word)) 271 | 272 | elif hook_with_extra_is_in_hooks(word, hooks): 273 | # word is like `somehook=data` and `somehook` is in our list of 274 | # potential hooks 275 | (hook_name, hook_extra) = word.split('=', 2) 276 | commands.append(IssueCommentCommand.hook(hook_name, hook_extra)) 277 | 278 | else: 279 | # First time we reach an unknown word, stop parsing. 280 | break 281 | 282 | return commands 283 | -------------------------------------------------------------------------------- /homu/html/queue.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Homu queue - {{repo_label}} {% if treeclosed %} [TREE CLOSED] {% endif %} 6 | 143 | 144 | 145 | {% if announcement != None %} 146 |
{{ announcement | safe }}
147 | {% endif %} 148 | 149 |

Homu queue - {% if repo_url %}{{repo_label}}{% else %}{{repo_label}}{% endif %} {% if treeclosed %} [TREE CLOSED below priority {{treeclosed}}] {% endif %}

150 | 151 |

152 | 153 |

154 | 155 |
156 |

This will create a new pull request consisting of 0 PRs.

157 |

A rollup is useful for shortening the queue, but jumping the queue is unfair to older PRs who have waited too long.

158 |

When creating a real rollup, see this instruction for reference.

159 |

160 | 161 | — 162 | 163 |

164 |
165 | 166 |

167 | {{ total }} total, {{ approved }} approved, {{ rolled_up }} rolled up, {{ failed }} failed 168 | / 169 | 170 | / 171 | 172 | 173 |

174 | 175 | 176 | 177 | 178 | 179 | 180 | {% if multiple %} 181 | 182 | {% endif %} 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | {% for state in states %} 197 | {% set checkbox_state = 198 | ('checked' if state.prechecked else '') if 199 | ((state.status == 'approved' or (state.status == 'pending' and not state.try_)) and state.rollup != 'never') 200 | else 'disabled' 201 | %} 202 | 203 | 204 | 205 | {% if multiple %} 206 | 207 | {% endif %} 208 | 209 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | {% endfor %} 225 | 226 |
Sort keyRepository#StatusMergeableTitleHead refAssigneeApproved byPriorityRollup
{{loop.index}}{{state.repo_label}}{{state.num}} 210 | {% if state.status == "pending" or state.status == "failure" or state.status == "success" %} 211 | {{state.status}}{{state.status_ext}} 212 | {% else %} 213 | {{state.status}}{{state.status_ext}} 214 | {% endif %} 215 | {{state.mergeable}}{{state.title}}{{state.head_ref}}{{state.assignee}}{{state.approved_by}}{{state.priority}}{{state.rollup}}
227 | 228 |

Open retry log

229 | 230 |

231 | 232 |

233 |

Caution: Synchronization has some caveats. Please follow the steps described in Fixing inconsistencies in the bors queue.

234 | 235 | 236 | 237 | 238 | 345 | 346 | 347 | -------------------------------------------------------------------------------- /homu/tests/test_parse_issue_comment.py: -------------------------------------------------------------------------------- 1 | from homu.parse_issue_comment import parse_issue_comment 2 | 3 | # Random commit number. Just so that we don't need to come up with a new one 4 | # for every test. 5 | commit = "5ffafdb1e94fa87334d4851a57564425e11a569e" 6 | other_commit = "4e4c9ddd781729173df2720d83e0f4d1b0102a94" 7 | 8 | 9 | def test_r_plus(): 10 | """ 11 | @bors r+ 12 | """ 13 | 14 | author = "jack" 15 | body = "@bors r+" 16 | commands = parse_issue_comment(author, body, commit, "bors") 17 | 18 | assert len(commands) == 1 19 | command = commands[0] 20 | assert command.action == 'approve' 21 | assert command.actor == 'jack' 22 | 23 | 24 | def test_r_plus_with_colon(): 25 | """ 26 | @bors: r+ 27 | """ 28 | 29 | author = "jack" 30 | body = "@bors: r+" 31 | commands = parse_issue_comment(author, body, commit, "bors") 32 | 33 | assert len(commands) == 1 34 | command = commands[0] 35 | assert command.action == 'approve' 36 | assert command.actor == 'jack' 37 | assert command.commit == commit 38 | 39 | 40 | def test_r_plus_with_sha(): 41 | """ 42 | @bors r+ {sha} 43 | """ 44 | 45 | author = "jack" 46 | body = "@bors r+ {}".format(other_commit) 47 | commands = parse_issue_comment(author, body, commit, "bors") 48 | 49 | assert len(commands) == 1 50 | command = commands[0] 51 | assert command.action == 'approve' 52 | assert command.actor == 'jack' 53 | assert command.commit == other_commit 54 | 55 | 56 | def test_r_equals(): 57 | """ 58 | @bors r=jill 59 | """ 60 | 61 | author = "jack" 62 | body = "@bors r=jill" 63 | commands = parse_issue_comment(author, body, commit, "bors") 64 | 65 | assert len(commands) == 1 66 | command = commands[0] 67 | assert command.action == 'approve' 68 | assert command.actor == 'jill' 69 | 70 | 71 | def test_r_equals_at_user(): 72 | """ 73 | @bors r=@jill 74 | """ 75 | 76 | author = "jack" 77 | body = "@bors r=@jill" 78 | commands = parse_issue_comment(author, body, commit, "bors") 79 | 80 | assert len(commands) == 1 81 | command = commands[0] 82 | assert command.action == 'approve' 83 | assert command.actor == 'jill' 84 | 85 | 86 | def test_hidden_r_equals(): 87 | author = "bors" 88 | body = """ 89 | :pushpin: Commit {0} has been approved by `jack` 90 | It is now in the [queue]({1}) for this repository.\n\n 91 | 92 | """.format(commit, "rust") 93 | 94 | commands = parse_issue_comment(author, body, commit, "bors") 95 | 96 | assert len(commands) == 1 97 | command = commands[0] 98 | assert command.action == 'approve' 99 | assert command.actor == 'jack' 100 | assert command.commit == commit 101 | 102 | 103 | def test_r_me(): 104 | """ 105 | Ignore r=me 106 | """ 107 | 108 | author = "jack" 109 | body = "@bors r=me" 110 | commands = parse_issue_comment(author, body, commit, "bors") 111 | 112 | # r=me is not a valid command, so no valid commands. 113 | assert len(commands) == 0 114 | 115 | 116 | def test_r_minus(): 117 | """ 118 | @bors r- 119 | """ 120 | 121 | author = "jack" 122 | body = "@bors r-" 123 | commands = parse_issue_comment(author, body, commit, "bors") 124 | 125 | assert len(commands) == 1 126 | command = commands[0] 127 | assert command.action == 'unapprove' 128 | 129 | 130 | def test_priority(): 131 | """ 132 | @bors p=5 133 | """ 134 | 135 | author = "jack" 136 | body = "@bors p=5" 137 | commands = parse_issue_comment(author, body, commit, "bors") 138 | 139 | assert len(commands) == 1 140 | command = commands[0] 141 | assert command.action == 'prioritize' 142 | assert command.priority == 5 143 | 144 | 145 | def test_approve_and_priority(): 146 | """ 147 | @bors r+ p=5 148 | """ 149 | 150 | author = "jack" 151 | body = "@bors r+ p=5" 152 | commands = parse_issue_comment(author, body, commit, "bors") 153 | 154 | assert len(commands) == 2 155 | approve_commands = [command for command in commands 156 | if command.action == 'approve'] 157 | prioritize_commands = [command for command in commands 158 | if command.action == 'prioritize'] 159 | assert len(approve_commands) == 1 160 | assert len(prioritize_commands) == 1 161 | 162 | assert approve_commands[0].actor == 'jack' 163 | assert prioritize_commands[0].priority == 5 164 | 165 | 166 | def test_approve_specific_and_priority(): 167 | """ 168 | @bors r+ {sha} p=5 169 | """ 170 | 171 | author = "jack" 172 | body = "@bors r+ {} p=5".format(other_commit) 173 | commands = parse_issue_comment(author, body, commit, "bors") 174 | 175 | assert len(commands) == 2 176 | approve_commands = [command for command in commands 177 | if command.action == 'approve'] 178 | prioritize_commands = [command for command in commands 179 | if command.action == 'prioritize'] 180 | assert len(approve_commands) == 1 181 | assert len(prioritize_commands) == 1 182 | 183 | assert approve_commands[0].actor == 'jack' 184 | assert approve_commands[0].commit == other_commit 185 | assert prioritize_commands[0].priority == 5 186 | 187 | 188 | def test_delegate_plus(): 189 | """ 190 | @bors delegate+ 191 | """ 192 | 193 | author = "jack" 194 | body = "@bors delegate+" 195 | commands = parse_issue_comment(author, body, commit, "bors") 196 | 197 | assert len(commands) == 1 198 | command = commands[0] 199 | assert command.action == 'delegate-author' 200 | 201 | 202 | def test_delegate_equals(): 203 | """ 204 | @bors delegate={username} 205 | """ 206 | 207 | author = "jack" 208 | body = "@bors delegate=jill" 209 | commands = parse_issue_comment(author, body, commit, "bors") 210 | 211 | assert len(commands) == 1 212 | command = commands[0] 213 | assert command.action == 'delegate' 214 | assert command.delegate_to == 'jill' 215 | 216 | 217 | def test_delegate_equals_at_user(): 218 | """ 219 | @bors delegate=@{username} 220 | """ 221 | 222 | author = "jack" 223 | body = "@bors delegate=@jill" 224 | commands = parse_issue_comment(author, body, commit, "bors") 225 | 226 | assert len(commands) == 1 227 | command = commands[0] 228 | assert command.action == 'delegate' 229 | assert command.delegate_to == 'jill' 230 | 231 | 232 | def test_delegate_minus(): 233 | """ 234 | @bors delegate- 235 | """ 236 | 237 | author = "jack" 238 | body = "@bors delegate-" 239 | commands = parse_issue_comment(author, body, commit, "bors") 240 | 241 | assert len(commands) == 1 242 | command = commands[0] 243 | assert command.action == 'undelegate' 244 | 245 | 246 | def test_retry(): 247 | """ 248 | @bors retry 249 | """ 250 | 251 | author = "jack" 252 | body = "@bors retry" 253 | commands = parse_issue_comment(author, body, commit, "bors") 254 | 255 | assert len(commands) == 1 256 | command = commands[0] 257 | assert command.action == 'retry' 258 | 259 | 260 | # def test_try(): 261 | # """ 262 | # @bors try 263 | # """ 264 | # 265 | # author = "jack" 266 | # body = "@bors try" 267 | # commands = parse_issue_comment(author, body, commit, "bors") 268 | # 269 | # assert len(commands) == 1 270 | # command = commands[0] 271 | # assert command.action == 'try' 272 | 273 | 274 | def test_try_minus(): 275 | """ 276 | @bors try- 277 | """ 278 | 279 | author = "jack" 280 | body = "@bors try-" 281 | commands = parse_issue_comment(author, body, commit, "bors") 282 | 283 | assert len(commands) == 0 284 | 285 | 286 | def test_rollup(): 287 | """ 288 | @bors rollup 289 | """ 290 | 291 | author = "jack" 292 | body = "@bors rollup" 293 | commands = parse_issue_comment(author, body, commit, "bors") 294 | 295 | assert len(commands) == 1 296 | command = commands[0] 297 | assert command.action == 'rollup' 298 | assert command.rollup_value == 1 299 | 300 | 301 | def test_rollup_minus(): 302 | """ 303 | @bors rollup- 304 | """ 305 | 306 | author = "jack" 307 | body = "@bors rollup-" 308 | commands = parse_issue_comment(author, body, commit, "bors") 309 | 310 | assert len(commands) == 1 311 | command = commands[0] 312 | assert command.action == 'rollup' 313 | assert command.rollup_value == 0 314 | 315 | 316 | def test_rollup_iffy(): 317 | """ 318 | @bors rollup=iffy 319 | """ 320 | 321 | author = "manishearth" 322 | body = "@bors rollup=iffy" 323 | commands = parse_issue_comment(author, body, commit, "bors") 324 | 325 | assert len(commands) == 1 326 | command = commands[0] 327 | assert command.action == 'rollup' 328 | assert command.rollup_value == -1 329 | 330 | 331 | def test_rollup_never(): 332 | """ 333 | @bors rollup=never 334 | """ 335 | 336 | author = "jack" 337 | body = "@bors rollup=never" 338 | commands = parse_issue_comment(author, body, commit, "bors") 339 | 340 | assert len(commands) == 1 341 | command = commands[0] 342 | assert command.action == 'rollup' 343 | assert command.rollup_value == -2 344 | 345 | 346 | def test_rollup_maybe(): 347 | """ 348 | @bors rollup=maybe 349 | """ 350 | 351 | author = "jack" 352 | body = "@bors rollup=maybe" 353 | commands = parse_issue_comment(author, body, commit, "bors") 354 | 355 | assert len(commands) == 1 356 | command = commands[0] 357 | assert command.action == 'rollup' 358 | assert command.rollup_value == 0 359 | 360 | 361 | def test_rollup_always(): 362 | """ 363 | @bors rollup=always 364 | """ 365 | 366 | author = "jack" 367 | body = "@bors rollup=always" 368 | commands = parse_issue_comment(author, body, commit, "bors") 369 | 370 | assert len(commands) == 1 371 | command = commands[0] 372 | assert command.action == 'rollup' 373 | assert command.rollup_value == 1 374 | 375 | 376 | def test_force(): 377 | """ 378 | @bors force 379 | """ 380 | 381 | author = "jack" 382 | body = "@bors force" 383 | commands = parse_issue_comment(author, body, commit, "bors") 384 | 385 | assert len(commands) == 1 386 | command = commands[0] 387 | assert command.action == 'force' 388 | 389 | 390 | def test_clean(): 391 | """ 392 | @bors clean 393 | """ 394 | 395 | author = "jack" 396 | body = "@bors clean" 397 | commands = parse_issue_comment(author, body, commit, "bors") 398 | 399 | assert len(commands) == 1 400 | command = commands[0] 401 | assert command.action == 'clean' 402 | 403 | 404 | def test_ping(): 405 | """ 406 | @bors ping 407 | """ 408 | 409 | author = "jack" 410 | body = "@bors ping" 411 | commands = parse_issue_comment(author, body, commit, "bors") 412 | 413 | assert len(commands) == 1 414 | command = commands[0] 415 | assert command.action == 'ping' 416 | assert command.ping_type == 'standard' 417 | 418 | 419 | def test_hello(): 420 | """ 421 | @bors hello? 422 | """ 423 | 424 | author = "jack" 425 | body = "@bors hello?" 426 | commands = parse_issue_comment(author, body, commit, "bors") 427 | 428 | assert len(commands) == 1 429 | command = commands[0] 430 | assert command.action == 'ping' 431 | assert command.ping_type == 'standard' 432 | 433 | 434 | def test_portal_ping(): 435 | """ 436 | @bors are you still there? 437 | """ 438 | 439 | author = "jack" 440 | body = "@bors are you still there?" 441 | commands = parse_issue_comment(author, body, commit, "bors") 442 | 443 | assert len(commands) == 1 444 | command = commands[0] 445 | assert command.action == 'ping' 446 | assert command.ping_type == 'portal' 447 | 448 | 449 | def test_treeclosed(): 450 | """ 451 | @bors treeclosed=50 452 | """ 453 | 454 | author = "jack" 455 | body = "@bors treeclosed=50" 456 | commands = parse_issue_comment(author, body, commit, "bors") 457 | 458 | assert len(commands) == 1 459 | command = commands[0] 460 | assert command.action == 'treeclosed' 461 | assert command.treeclosed_value == 50 462 | 463 | 464 | def test_treeclosed_minus(): 465 | """ 466 | @bors treeclosed- 467 | """ 468 | 469 | author = "jack" 470 | body = "@bors treeclosed-" 471 | commands = parse_issue_comment(author, body, commit, "bors") 472 | 473 | assert len(commands) == 1 474 | command = commands[0] 475 | assert command.action == 'untreeclosed' 476 | 477 | 478 | def test_hook(): 479 | """ 480 | Test hooks that are defined in the configuration 481 | 482 | @bors secondhook 483 | """ 484 | 485 | author = "jack" 486 | body = "@bors secondhook" 487 | commands = parse_issue_comment( 488 | author, body, commit, "bors", 489 | ['firsthook', 'secondhook', 'thirdhook']) 490 | 491 | assert len(commands) == 1 492 | command = commands[0] 493 | assert command.action == 'hook' 494 | assert command.hook_name == 'secondhook' 495 | assert command.hook_extra is None 496 | 497 | 498 | def test_hook_equals(): 499 | """ 500 | Test hooks that are defined in the configuration 501 | 502 | @bors secondhook=extra 503 | """ 504 | 505 | author = "jack" 506 | body = "@bors secondhook=extra" 507 | commands = parse_issue_comment( 508 | author, body, commit, "bors", 509 | ['firsthook', 'secondhook', 'thirdhook']) 510 | 511 | assert len(commands) == 1 512 | command = commands[0] 513 | assert command.action == 'hook' 514 | assert command.hook_name == 'secondhook' 515 | assert command.hook_extra == 'extra' 516 | 517 | 518 | def test_multiple_hooks(): 519 | """ 520 | Test hooks that are defined in the configuration 521 | 522 | @bors thirdhook secondhook=extra 523 | """ 524 | 525 | author = "jack" 526 | body = "@bors thirdhook secondhook=extra" 527 | commands = parse_issue_comment( 528 | author, body, commit, "bors", 529 | ['firsthook', 'secondhook', 'thirdhook']) 530 | 531 | assert len(commands) == 2 532 | secondhook_commands = [command for command in commands 533 | if command.action == 'hook' 534 | and command.hook_name == 'secondhook'] 535 | thirdhook_commands = [command for command in commands 536 | if command.action == 'hook' 537 | and command.hook_name == 'thirdhook'] 538 | assert len(secondhook_commands) == 1 539 | assert len(thirdhook_commands) == 1 540 | assert secondhook_commands[0].hook_extra == 'extra' 541 | assert thirdhook_commands[0].hook_extra is None 542 | 543 | 544 | def test_similar_name(): 545 | """ 546 | Test that a username that starts with 'bors' doesn't trigger. 547 | """ 548 | 549 | author = "jack" 550 | body = """ 551 | @bors-servo r+ 552 | """ 553 | commands = parse_issue_comment(author, body, commit, "bors") 554 | 555 | assert len(commands) == 0 556 | 557 | 558 | def test_parse_up_to_first_unknown_word(): 559 | """ 560 | Test that when parsing, once we arrive at an unknown word, we stop parsing 561 | """ 562 | 563 | author = "jack" 564 | body = """ 565 | @bors retry -- yielding priority to the rollup 566 | """ 567 | commands = parse_issue_comment(author, body, commit, "bors") 568 | 569 | assert len(commands) == 1 570 | command = commands[0] 571 | assert command.action == 'retry' 572 | 573 | body = """ 574 | @bors retry (yielding priority to the rollup) 575 | """ 576 | commands = parse_issue_comment(author, body, commit, "bors") 577 | 578 | assert len(commands) == 1 579 | command = commands[0] 580 | assert command.action == 'retry' 581 | 582 | 583 | def test_ignore_commands_before_bors_line(): 584 | """ 585 | Test that when command-like statements appear before the @bors part, 586 | they don't get parsed 587 | """ 588 | 589 | author = "jack" 590 | body = """ 591 | A sentence that includes command-like statements, like r- or ping or delegate+ or the like. 592 | 593 | @bors r+ 594 | """ # noqa 595 | commands = parse_issue_comment(author, body, commit, "bors") 596 | 597 | assert len(commands) == 1 598 | command = commands[0] 599 | assert command.action == 'approve' 600 | assert command.actor == 'jack' 601 | 602 | 603 | def test_ignore_commands_after_bors_line(): 604 | """ 605 | Test that when command-like statements appear after the @bors part, 606 | they don't get parsed 607 | """ 608 | 609 | author = "jack" 610 | body = """ 611 | @bors r+ 612 | 613 | A sentence that includes command-like statements, like r- or ping or delegate+ or the like. 614 | """ # noqa 615 | commands = parse_issue_comment(author, body, commit, "bors") 616 | 617 | assert len(commands) == 1 618 | command = commands[0] 619 | assert command.action == 'approve' 620 | assert command.actor == 'jack' 621 | 622 | 623 | def test_in_quote(): 624 | """ 625 | Test that a command in a quote (e.g. when replying by e-mail) doesn't 626 | trigger. 627 | """ 628 | 629 | author = "jack" 630 | body = """ 631 | > @bors r+ 632 | """ 633 | commands = parse_issue_comment(author, body, commit, "bors") 634 | 635 | assert len(commands) == 0 636 | -------------------------------------------------------------------------------- /homu/server.py: -------------------------------------------------------------------------------- 1 | import hmac 2 | import json 3 | import urllib.parse 4 | from .main import ( 5 | PullReqState, 6 | parse_commands, 7 | db_query, 8 | IGNORE_BLOCK_END, 9 | IGNORE_BLOCK_START, 10 | INTERRUPTED_BY_HOMU_RE, 11 | suppress_ignore_block, 12 | suppress_pings, 13 | synchronize, 14 | LabelEvent, 15 | ) 16 | from . import comments 17 | from . import utils 18 | from .utils import lazy_debug 19 | import github3 20 | import jinja2 21 | import requests 22 | import pkg_resources 23 | from bottle import ( 24 | get, 25 | post, 26 | run, 27 | request, 28 | redirect, 29 | abort, 30 | response, 31 | error, 32 | ) 33 | from threading import Thread 34 | import sys 35 | import os 36 | import traceback 37 | from retrying import retry 38 | import random 39 | import string 40 | import time 41 | 42 | import bottle 43 | bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 10 44 | 45 | 46 | class G: 47 | pass 48 | 49 | 50 | g = G() 51 | 52 | 53 | ROLLUP_STR = { 54 | -2: 'never', 55 | -1: 'iffy', 56 | 0: '', 57 | 1: 'always', 58 | } 59 | 60 | 61 | def find_state(sha): 62 | for repo_label, repo_states in g.states.items(): 63 | for state in repo_states.values(): 64 | if state.merge_sha == sha: 65 | return state, repo_label 66 | 67 | raise ValueError('Invalid SHA') 68 | 69 | 70 | def get_repo(repo_label, repo_cfg): 71 | repo = g.repos[repo_label].gh 72 | if not repo: 73 | repo = g.gh.repository(repo_cfg['owner'], repo_cfg['name']) 74 | g.repos[repo_label].gh = repo 75 | assert repo.owner.login == repo_cfg['owner'] 76 | assert repo.name == repo_cfg['name'] 77 | return repo 78 | 79 | 80 | @get('/') 81 | def index(): 82 | return g.tpls['index'].render(repos=[g.repos[label] 83 | for label in sorted(g.repos)]) 84 | 85 | 86 | @get('/results//') 87 | def result(repo_label, pull): 88 | if repo_label not in g.states: 89 | abort(404, 'No such repository: {}'.format(repo_label)) 90 | states = [state for state in g.states[repo_label].values() 91 | if state.num == pull] 92 | if len(states) == 0: 93 | abort(404, 'No build results for pull request {}'.format(pull)) 94 | 95 | state = states[0] 96 | builders = [] 97 | repo_url = 'https://github.com/{}/{}'.format( 98 | g.cfg['repo'][repo_label]['owner'], 99 | g.cfg['repo'][repo_label]['name']) 100 | for (builder, data) in state.build_res.items(): 101 | result = "pending" 102 | if data['res'] is not None: 103 | result = "success" if data['res'] else "failed" 104 | 105 | builder_details = { 106 | 'result': result, 107 | 'name': builder, 108 | } 109 | 110 | if data['url']: 111 | builder_details['url'] = data['url'] 112 | 113 | builders.append(builder_details) 114 | 115 | return g.tpls['build_res'].render(repo_label=repo_label, repo_url=repo_url, 116 | builders=builders, pull=pull) 117 | 118 | 119 | @get('/queue/') 120 | def queue(repo_label): 121 | if repo_label not in g.cfg['repo'] and repo_label != 'all': 122 | abort(404) 123 | 124 | logger = g.logger.getChild('queue') 125 | 126 | lazy_debug(logger, lambda: 'repo_label: {}'.format(repo_label)) 127 | 128 | single_repo_closed = None 129 | treeclosed_src = None 130 | if repo_label == 'all': 131 | labels = g.repos.keys() 132 | multiple = True 133 | repo_url = None 134 | else: 135 | labels = repo_label.split('+') 136 | multiple = len(labels) > 1 137 | if repo_label in g.repos and g.repos[repo_label].treeclosed >= 0: 138 | single_repo_closed = g.repos[repo_label].treeclosed 139 | treeclosed_src = g.repos[repo_label].treeclosed_src 140 | repo_url = 'https://github.com/{}/{}'.format( 141 | g.cfg['repo'][repo_label]['owner'], 142 | g.cfg['repo'][repo_label]['name']) 143 | 144 | states = [] 145 | for label in labels: 146 | try: 147 | states += g.states[label].values() 148 | except KeyError: 149 | abort(404, 'No such repository: {}'.format(label)) 150 | 151 | prechecked_prs = set() 152 | if request.query.get('prs'): 153 | prechecked_prs = set(request.query.get('prs').split(',')) 154 | 155 | pull_states = sorted(states) 156 | rows = [] 157 | for state in pull_states: 158 | treeclosed = (single_repo_closed and 159 | state.priority < g.repos[state.repo_label].treeclosed) 160 | status_ext = '' 161 | 162 | if state.try_: 163 | status_ext += ' (try)' 164 | 165 | rows.append({ 166 | 'status': state.get_status(), 167 | 'status_ext': status_ext, 168 | 'priority': state.priority, 169 | 'rollup': ROLLUP_STR.get(state.rollup, ''), 170 | 'prechecked': str(state.num) in prechecked_prs, 171 | 'url': 'https://github.com/{}/{}/pull/{}'.format(state.owner, 172 | state.name, 173 | state.num), 174 | 'num': state.num, 175 | 'approved_by': state.approved_by, 176 | 'title': state.title, 177 | 'head_ref': state.head_ref, 178 | 'mergeable': ('yes' if state.mergeable is True else 179 | 'no' if state.mergeable is False else ''), 180 | 'assignee': state.assignee, 181 | 'repo_label': state.repo_label, 182 | 'repo_url': 'https://github.com/{}/{}'.format(state.owner, 183 | state.name), 184 | 'greyed': "treeclosed" if treeclosed else "", 185 | }) 186 | 187 | return g.tpls['queue'].render( 188 | repo_url=repo_url, 189 | repo_label=repo_label, 190 | treeclosed=single_repo_closed, 191 | treeclosed_src=treeclosed_src, 192 | states=rows, 193 | oauth_client_id=g.cfg['github']['app_client_id'], 194 | total=len(pull_states), 195 | approved=len([x for x in pull_states if x.approved_by]), 196 | rolled_up=len([x for x in pull_states if x.rollup > 0]), 197 | failed=len([x for x in pull_states if x.status == 'failure' or 198 | x.status == 'error']), 199 | multiple=multiple, 200 | ) 201 | 202 | 203 | @get('/retry_log/') 204 | def retry_log(repo_label): 205 | if repo_label not in g.cfg['repo']: 206 | abort(404) 207 | 208 | logger = g.logger.getChild('retry_log') 209 | 210 | lazy_debug(logger, lambda: 'repo_label: {}'.format(repo_label)) 211 | 212 | repo_url = 'https://github.com/{}/{}'.format( 213 | g.cfg['repo'][repo_label]['owner'], 214 | g.cfg['repo'][repo_label]['name'], 215 | ) 216 | 217 | db_query( 218 | g.db, 219 | ''' 220 | SELECT num, time, src, msg FROM retry_log 221 | WHERE repo = ? ORDER BY time DESC 222 | ''', 223 | [repo_label], 224 | ) 225 | logs = [ 226 | {'num': num, 'time': time, 'src': src, 'msg': msg} 227 | for num, time, src, msg in g.db.fetchall() 228 | ] 229 | 230 | return g.tpls['retry_log'].render( 231 | repo_url=repo_url, 232 | repo_label=repo_label, 233 | logs=logs, 234 | ) 235 | 236 | 237 | @get('/callback') 238 | def callback(): 239 | logger = g.logger.getChild('callback') 240 | 241 | response.content_type = 'text/plain' 242 | 243 | code = request.query.code 244 | state = json.loads(request.query.state) 245 | 246 | lazy_debug(logger, lambda: 'state: {}'.format(state)) 247 | oauth_url = 'https://github.com/login/oauth/access_token' 248 | 249 | try: 250 | res = requests.post(oauth_url, data={ 251 | 'client_id': g.cfg['github']['app_client_id'], 252 | 'client_secret': g.cfg['github']['app_client_secret'], 253 | 'code': code, 254 | }) 255 | except Exception as ex: 256 | logger.warn('/callback encountered an error ' 257 | 'during github oauth callback') 258 | lazy_debug( 259 | logger, 260 | lambda ex=ex: 'github oauth callback err: {}'.format(ex), 261 | ) 262 | abort(502, 'Bad Gateway') 263 | 264 | args = urllib.parse.parse_qs(res.text) 265 | token = args['access_token'][0] 266 | 267 | repo_label = state['repo_label'] 268 | repo_cfg = g.repo_cfgs[repo_label] 269 | repo = get_repo(repo_label, repo_cfg) 270 | 271 | user_gh = github3.login(token=token) 272 | 273 | if state['cmd'] == 'rollup': 274 | return rollup(user_gh, state, repo_label, repo_cfg, repo) 275 | elif state['cmd'] == 'synch': 276 | return synch(user_gh, state, repo_label, repo_cfg, repo) 277 | else: 278 | abort(400, 'Invalid command') 279 | 280 | 281 | def rollup(user_gh, state, repo_label, repo_cfg, repo): 282 | user_repo = user_gh.repository(user_gh.user().login, repo.name) 283 | if user_repo is None: 284 | return 'You must have a fork of rust-lang/rust named rust under your user account.' # noqa 285 | base_repo = user_gh.repository(repo.owner.login, repo.name) 286 | 287 | nums = state.get('nums', []) 288 | if nums: 289 | try: 290 | rollup_states = [g.states[repo_label][num] for num in nums] 291 | except KeyError as e: 292 | return 'Invalid PR number: {}'.format(e.args[0]) 293 | else: 294 | rollup_states = [x for x in g.states[repo_label].values() if x.rollup] 295 | rollup_states = [x for x in rollup_states if x.approved_by] 296 | rollup_states.sort(key=lambda x: x.num) 297 | 298 | if not rollup_states: 299 | return 'No pull requests are marked as rollup' 300 | 301 | base_ref = rollup_states[0].base_ref 302 | 303 | base_sha = repo.ref('heads/' + base_ref).object.sha 304 | branch_name = 'rollup-' + ''.join( 305 | random.choice(string.digits + string.ascii_lowercase) for _ in range(7) 306 | ) 307 | utils.github_set_ref( 308 | user_repo, 309 | 'heads/' + branch_name, 310 | base_sha, 311 | force=True, 312 | ) 313 | 314 | successes = [] 315 | failures = [] 316 | 317 | for state in rollup_states: 318 | if base_ref != state.base_ref: 319 | failures.append(state) 320 | continue 321 | 322 | state.body = suppress_pings(state.body or "") 323 | state.body = suppress_ignore_block(state.body) 324 | 325 | merge_msg = 'Rollup merge of #{} - {}, r={}\n\n{}\n\n{}'.format( 326 | state.num, 327 | state.head_ref, 328 | state.approved_by, 329 | state.title, 330 | state.body, 331 | ) 332 | 333 | try: 334 | user_repo.merge(branch_name, state.head_sha, merge_msg) 335 | except github3.models.GitHubError as e: 336 | if e.code != 409: 337 | raise 338 | 339 | failures.append(state) 340 | else: 341 | successes.append(state) 342 | 343 | title = 'Rollup of {} pull requests'.format(len(successes)) 344 | 345 | body = 'Successful merges:\n\n' 346 | for x in successes: 347 | body += ' - #{} ({})\n'.format(x.num, x.title) 348 | 349 | if len(failures) != 0: 350 | body += '\nFailed merges:\n\n' 351 | for x in failures: 352 | body += ' - #{} ({})\n'.format(x.num, x.title) 353 | body += '\nr? @ghost\n@rustbot modify labels: rollup' 354 | 355 | # Set web.base_url in cfg to enable 356 | base_url = g.cfg['web'].get('base_url') 357 | if not base_url: 358 | # If web.base_url is not present, fall back to using web.canonical_url 359 | base_url = g.cfg['web'].get('canonical_url') 360 | 361 | if base_url: 362 | pr_list = ','.join(str(x.num) for x in successes) 363 | link = '{}/queue/{}?prs={}'.format(base_url, repo_label, pr_list) 364 | body += '\n' 365 | body += IGNORE_BLOCK_START 366 | body += '\n[Create a similar rollup]({})\n'.format(link) 367 | body += IGNORE_BLOCK_END 368 | 369 | try: 370 | pull = base_repo.create_pull( 371 | title, 372 | state.base_ref, 373 | user_repo.owner.login + ':' + branch_name, 374 | body, 375 | ) 376 | except github3.models.GitHubError as e: 377 | return e.response.text 378 | else: 379 | redirect(pull.html_url) 380 | 381 | 382 | @post('/github') 383 | def github(): 384 | logger = g.logger.getChild('github') 385 | 386 | response.content_type = 'text/plain' 387 | 388 | payload = request.body.read() 389 | info = request.json 390 | 391 | lazy_debug(logger, lambda: 'info: {}'.format(utils.remove_url_keys_from_json(info))) # noqa 392 | 393 | owner_info = info['repository']['owner'] 394 | owner = owner_info.get('login') or owner_info['name'] 395 | repo_label = g.repo_labels[owner, info['repository']['name']] 396 | repo_cfg = g.repo_cfgs[repo_label] 397 | 398 | hmac_method, hmac_sig = request.headers['X-Hub-Signature'].split('=') 399 | if hmac_sig != hmac.new( 400 | repo_cfg['github']['secret'].encode('utf-8'), 401 | payload, 402 | hmac_method, 403 | ).hexdigest(): 404 | abort(400, 'Invalid signature') 405 | 406 | event_type = request.headers['X-Github-Event'] 407 | 408 | if event_type == 'pull_request_review_comment': 409 | action = info['action'] 410 | original_commit_id = info['comment']['original_commit_id'] 411 | head_sha = info['pull_request']['head']['sha'] 412 | 413 | if action == 'created' and original_commit_id == head_sha: 414 | pull_num = info['pull_request']['number'] 415 | body = info['comment']['body'] 416 | username = info['sender']['login'] 417 | user_id = info['sender']['id'] 418 | 419 | state = g.states[repo_label].get(pull_num) 420 | if state: 421 | state.title = info['pull_request']['title'] 422 | state.body = info['pull_request']['body'] 423 | 424 | if parse_commands( 425 | body, 426 | username, 427 | user_id, 428 | repo_label, 429 | repo_cfg, 430 | state, 431 | g.my_username, 432 | g.db, 433 | g.states, 434 | realtime=True, 435 | sha=original_commit_id, 436 | command_src=info['comment']['html_url'], 437 | ): 438 | state.save() 439 | 440 | g.queue_handler() 441 | 442 | elif event_type == 'pull_request': 443 | action = info['action'] 444 | pull_num = info['number'] 445 | head_sha = info['pull_request']['head']['sha'] 446 | 447 | if action == 'synchronize': 448 | state = g.states[repo_label][pull_num] 449 | state.head_advanced(head_sha) 450 | 451 | state.save() 452 | 453 | elif action in ['opened', 'reopened']: 454 | state = PullReqState(pull_num, head_sha, '', g.db, repo_label, 455 | g.mergeable_que, g.gh, 456 | info['repository']['owner']['login'], 457 | info['repository']['name'], 458 | repo_cfg.get('labels', {}), 459 | g.repos, 460 | repo_cfg.get('test-on-fork')) 461 | state.title = info['pull_request']['title'] 462 | state.body = info['pull_request']['body'] 463 | state.head_ref = info['pull_request']['head']['repo']['owner']['login'] + ':' + info['pull_request']['head']['ref'] # noqa 464 | state.base_ref = info['pull_request']['base']['ref'] 465 | state.set_mergeable(info['pull_request']['mergeable']) 466 | state.assignee = (info['pull_request']['assignee']['login'] if 467 | info['pull_request']['assignee'] else '') 468 | 469 | found = False 470 | 471 | if action == 'reopened': 472 | # FIXME: Review comments are ignored here 473 | for c in state.get_repo().issue(pull_num).iter_comments(): 474 | found = parse_commands( 475 | c.body, 476 | c.user.login, 477 | c.user.id, 478 | repo_label, 479 | repo_cfg, 480 | state, 481 | g.my_username, 482 | g.db, 483 | g.states, 484 | command_src=c.to_json()['html_url'], 485 | # FIXME switch to `c.html_url` 486 | # after updating github3 to 1.3.0+ 487 | ) or found 488 | 489 | status = '' 490 | for info in utils.github_iter_statuses(state.get_repo(), 491 | state.head_sha): 492 | if info.context == 'homu': 493 | status = info.state 494 | break 495 | 496 | state.set_status(status) 497 | 498 | state.save() 499 | 500 | g.states[repo_label][pull_num] = state 501 | 502 | if found: 503 | g.queue_handler() 504 | 505 | elif action == 'closed': 506 | state = g.states[repo_label][pull_num] 507 | if hasattr(state, 'fake_merge_sha'): 508 | def inner(): 509 | utils.github_set_ref( 510 | state.get_repo(), 511 | 'heads/' + state.base_ref, 512 | state.merge_sha, 513 | force=True, 514 | ) 515 | 516 | def fail(err): 517 | state.add_comment(':boom: Failed to recover from the ' 518 | 'artificial commit. See {} for details.' 519 | ' ({})'.format(state.fake_merge_sha, 520 | err)) 521 | 522 | utils.retry_until(inner, fail, state) 523 | 524 | del g.states[repo_label][pull_num] 525 | 526 | db_query(g.db, 'DELETE FROM pull WHERE repo = ? AND num = ?', 527 | [repo_label, pull_num]) 528 | db_query(g.db, 'DELETE FROM build_res WHERE repo = ? AND num = ?', 529 | [repo_label, pull_num]) 530 | db_query(g.db, 'DELETE FROM mergeable WHERE repo = ? AND num = ?', 531 | [repo_label, pull_num]) 532 | 533 | g.queue_handler() 534 | 535 | elif action in ['assigned', 'unassigned']: 536 | state = g.states[repo_label][pull_num] 537 | state.assignee = (info['pull_request']['assignee']['login'] if 538 | info['pull_request']['assignee'] else '') 539 | 540 | state.save() 541 | 542 | elif action == 'edited': 543 | state = g.states[repo_label][pull_num] 544 | 545 | base_ref = info['pull_request']['base']['ref'] 546 | if state.base_ref != base_ref: 547 | state.base_ref = base_ref 548 | state.set_mergeable(None) 549 | # Remove PR approval when the branch changes, to prevent the PR 550 | # authors to merge the changes on other branches 551 | if state.get_status() != '': 552 | state.approved_by = '' 553 | state.set_status('') 554 | state.change_labels(LabelEvent.PUSHED) 555 | state.add_comment( 556 | ':warning: The base branch changed to `{}`, and the ' 557 | 'PR will need to be re-approved.\n\n' 558 | ''.format(base_ref, g.my_username) 559 | ) 560 | 561 | state.title = info['pull_request']['title'] 562 | state.body = info['pull_request']['body'] 563 | 564 | state.save() 565 | 566 | else: 567 | lazy_debug(logger, lambda: 'Invalid pull_request action: {}'.format(action)) # noqa 568 | 569 | elif event_type == 'push': 570 | ref = info['ref'][len('refs/heads/'):] 571 | 572 | for state in list(g.states[repo_label].values()): 573 | if state.base_ref == ref: 574 | state.set_mergeable(None, cause={ 575 | 'sha': info['head_commit']['id'], 576 | 'title': info['head_commit']['message'].splitlines()[0], 577 | }) 578 | 579 | if state.head_sha == info['before']: 580 | if state.status: 581 | state.change_labels(LabelEvent.PUSHED) 582 | state.head_advanced(info['after']) 583 | 584 | state.save() 585 | 586 | elif event_type == 'issue_comment': 587 | action = info['action'] 588 | body = info['comment']['body'] 589 | username = info['comment']['user']['login'] 590 | user_id = info['comment']['user']['id'] 591 | pull_num = info['issue']['number'] 592 | 593 | state = g.states[repo_label].get(pull_num) 594 | 595 | if action == 'created' and 'pull_request' in info['issue'] and state: 596 | state.title = info['issue']['title'] 597 | state.body = info['issue']['body'] 598 | 599 | if parse_commands( 600 | body, 601 | username, 602 | user_id, 603 | repo_label, 604 | repo_cfg, 605 | state, 606 | g.my_username, 607 | g.db, 608 | g.states, 609 | realtime=True, 610 | command_src=info['comment']['html_url'], 611 | ): 612 | state.save() 613 | 614 | g.queue_handler() 615 | 616 | elif event_type == 'status': 617 | try: 618 | state, repo_label = find_state(info['sha']) 619 | except ValueError: 620 | return 'OK' 621 | 622 | status_name = "" 623 | if 'status' in repo_cfg: 624 | for name, value in repo_cfg['status'].items(): 625 | if 'context' in value and value['context'] == info['context']: 626 | status_name = name 627 | if status_name == "": 628 | return 'OK' 629 | 630 | if info['state'] == 'pending': 631 | return 'OK' 632 | 633 | for row in info['branches']: 634 | if row['name'] == state.base_ref: 635 | return 'OK' 636 | 637 | report_build_res(info['state'] == 'success', info['target_url'], 638 | 'status-' + status_name, state, logger, repo_cfg) 639 | 640 | elif event_type == 'check_run': 641 | try: 642 | state, repo_label = find_state(info['check_run']['head_sha']) 643 | except ValueError: 644 | return 'OK' 645 | 646 | current_run_name = info['check_run']['name'] 647 | checks_name = None 648 | if 'checks' in repo_cfg: 649 | for name, value in repo_cfg['checks'].items(): 650 | if state.try_ and 'try_name' in value: 651 | if value['try_name'] == current_run_name: 652 | checks_name = name 653 | elif 'name' in value and value['name'] == current_run_name: 654 | checks_name = name 655 | if checks_name is None: 656 | return 'OK' 657 | 658 | if info['check_run']['status'] != 'completed': 659 | return 'OK' 660 | if info['check_run']['conclusion'] is None: 661 | return 'OK' 662 | # GHA marks jobs as skipped, if they are not run due to the job 663 | # condition. This prevents bors from failing because of these jobs. 664 | if info['check_run']['conclusion'] == 'skipped': 665 | return 'OK' 666 | 667 | report_build_res( 668 | info['check_run']['conclusion'] == 'success', 669 | info['check_run']['details_url'], 670 | 'checks-' + checks_name, 671 | state, logger, repo_cfg, 672 | ) 673 | 674 | return 'OK' 675 | 676 | 677 | def report_build_res(succ, url, builder, state, logger, repo_cfg): 678 | lazy_debug(logger, 679 | lambda: 'build result {}: builder = {}, succ = {}, current build_res = {}' # noqa 680 | .format(state, builder, succ, 681 | state.build_res_summary())) 682 | 683 | state.set_build_res(builder, succ, url) 684 | 685 | if succ: 686 | if all(x['res'] for x in state.build_res.values()): 687 | state.set_status('success') 688 | utils.github_create_status( 689 | state.get_repo(), state.head_sha, 690 | 'success', url, "Test successful", context='homu' 691 | ) 692 | 693 | if state.approved_by and not state.try_: 694 | # The set_ref call below sometimes fails with 422 failed to 695 | # fast forward. We believe this is a spurious error on GitHub's 696 | # side, though it's not entirely clear why. We sleep for 1 697 | # minute before trying it after setting the status to try to 698 | # increase the likelihood it will work, and also retry the 699 | # set_ref a few times. 700 | time.sleep(60) 701 | state.add_comment(comments.BuildCompleted( 702 | approved_by=state.approved_by, 703 | base_ref=state.base_ref, 704 | builders={k: v["url"] for k, v in state.build_res.items()}, 705 | merge_sha=state.merge_sha, 706 | )) 707 | state.change_labels(LabelEvent.SUCCEED) 708 | 709 | def set_ref_inner(): 710 | utils.github_set_ref(state.get_repo(), 'heads/' + 711 | state.base_ref, state.merge_sha) 712 | if state.test_on_fork is not None: 713 | utils.github_set_ref(state.get_test_on_fork_repo(), 714 | 'heads/' + state.base_ref, 715 | state.merge_sha, force=True) 716 | 717 | def set_ref(): 718 | try: 719 | set_ref_inner() 720 | except github3.models.GitHubError: 721 | utils.github_create_status( 722 | state.get_repo(), 723 | state.merge_sha, 724 | 'success', '', 725 | 'Branch protection bypassed', 726 | context='homu') 727 | set_ref_inner() 728 | 729 | error = None 730 | for i in range(0, 5): 731 | try: 732 | set_ref() 733 | state.fake_merge(repo_cfg) 734 | error = None 735 | except github3.models.GitHubError as e: 736 | error = e 737 | pass 738 | if error is None: 739 | break 740 | else: 741 | time.sleep(10) 742 | 743 | if error is not None: 744 | state.set_status('error') 745 | desc = ('Test was successful, but fast-forwarding failed:' 746 | ' {}'.format(error)) 747 | utils.github_create_status(state.get_repo(), 748 | state.head_sha, 'error', url, 749 | desc, context='homu') 750 | 751 | state.add_comment(':eyes: ' + desc) 752 | else: 753 | state.add_comment(comments.TryBuildCompleted( 754 | builders={k: v["url"] for k, v in state.build_res.items()}, 755 | merge_sha=state.merge_sha, 756 | )) 757 | state.change_labels(LabelEvent.TRY_SUCCEED) 758 | 759 | else: 760 | if state.status == 'pending': 761 | state.set_status('failure') 762 | utils.github_create_status( 763 | state.get_repo(), state.head_sha, 764 | 'failure', url, "Test failed", context='homu' 765 | ) 766 | 767 | if state.try_: 768 | state.add_comment(comments.TryBuildFailed( 769 | builder_url=url, 770 | builder_name=builder, 771 | )) 772 | state.change_labels(LabelEvent.TRY_FAILED) 773 | else: 774 | state.add_comment(comments.BuildFailed( 775 | builder_url=url, 776 | builder_name=builder, 777 | )) 778 | state.change_labels(LabelEvent.FAILED) 779 | 780 | g.queue_handler() 781 | 782 | 783 | @post('/buildbot') 784 | def buildbot(): 785 | logger = g.logger.getChild('buildbot') 786 | 787 | response.content_type = 'text/plain' 788 | 789 | for row in json.loads(request.forms.packets): 790 | if row['event'] == 'buildFinished': 791 | info = row['payload']['build'] 792 | lazy_debug(logger, lambda: 'info: {}'.format(info)) 793 | props = dict(x[:2] for x in info['properties']) 794 | 795 | if 'retry' in info['text']: 796 | continue 797 | 798 | if not props['revision']: 799 | continue 800 | 801 | try: 802 | state, repo_label = find_state(props['revision']) 803 | except ValueError: 804 | lazy_debug(logger, 805 | lambda: 'Invalid commit ID from Buildbot: {}'.format(props['revision'])) # noqa 806 | continue 807 | 808 | lazy_debug(logger, lambda: 'state: {}, {}'.format(state, state.build_res_summary())) # noqa 809 | 810 | if info['builderName'] not in state.build_res: 811 | lazy_debug(logger, 812 | lambda: 'Invalid builder from Buildbot: {}'.format(info['builderName'])) # noqa 813 | continue 814 | 815 | repo_cfg = g.repo_cfgs[repo_label] 816 | 817 | if request.forms.secret != repo_cfg['buildbot']['secret']: 818 | abort(400, 'Invalid secret') 819 | 820 | build_succ = 'successful' in info['text'] or info['results'] == 0 821 | 822 | url = '{}/builders/{}/builds/{}'.format( 823 | repo_cfg['buildbot']['url'], 824 | info['builderName'], 825 | props['buildnumber'], 826 | ) 827 | 828 | if 'interrupted' in info['text']: 829 | step_name = '' 830 | for step in reversed(info['steps']): 831 | if 'interrupted' in step.get('text', []): 832 | step_name = step['name'] 833 | break 834 | 835 | if step_name: 836 | try: 837 | url = ('{}/builders/{}/builds/{}/steps/{}/logs/interrupt' # noqa 838 | ).format(repo_cfg['buildbot']['url'], 839 | info['builderName'], 840 | props['buildnumber'], 841 | step_name,) 842 | res = requests.get(url) 843 | except Exception as ex: 844 | logger.warn('/buildbot encountered an error during ' 845 | 'github logs request') 846 | lazy_debug( 847 | logger, 848 | lambda ex=ex: 'buildbot logs err: {}'.format(ex), 849 | ) 850 | abort(502, 'Bad Gateway') 851 | 852 | mat = INTERRUPTED_BY_HOMU_RE.search(res.text) 853 | if mat: 854 | interrupt_token = mat.group(1) 855 | if getattr(state, 'interrupt_token', 856 | '') != interrupt_token: 857 | state.interrupt_token = interrupt_token 858 | 859 | if state.status == 'pending': 860 | state.set_status('') 861 | 862 | desc = (':snowman: The build was interrupted ' 863 | 'to prioritize another pull request.') 864 | state.add_comment(desc) 865 | state.change_labels(LabelEvent.INTERRUPTED) 866 | utils.github_create_status(state.get_repo(), 867 | state.head_sha, 868 | 'error', url, 869 | desc, 870 | context='homu') 871 | 872 | g.queue_handler() 873 | 874 | continue 875 | 876 | else: 877 | logger.error('Corrupt payload from Buildbot') 878 | 879 | report_build_res(build_succ, url, info['builderName'], 880 | state, logger, repo_cfg) 881 | 882 | elif row['event'] == 'buildStarted': 883 | info = row['payload']['build'] 884 | lazy_debug(logger, lambda: 'info: {}'.format(info)) 885 | props = dict(x[:2] for x in info['properties']) 886 | 887 | if not props['revision']: 888 | continue 889 | 890 | try: 891 | state, repo_label = find_state(props['revision']) 892 | except ValueError: 893 | pass 894 | else: 895 | if info['builderName'] in state.build_res: 896 | repo_cfg = g.repo_cfgs[repo_label] 897 | 898 | if request.forms.secret != repo_cfg['buildbot']['secret']: 899 | abort(400, 'Invalid secret') 900 | 901 | url = '{}/builders/{}/builds/{}'.format( 902 | repo_cfg['buildbot']['url'], 903 | info['builderName'], 904 | props['buildnumber'], 905 | ) 906 | 907 | state.set_build_res(info['builderName'], None, url) 908 | 909 | if g.buildbot_slots[0] == props['revision']: 910 | g.buildbot_slots[0] = '' 911 | 912 | g.queue_handler() 913 | 914 | return 'OK' 915 | 916 | 917 | @get('/assets/') 918 | def server_static(file): 919 | current_path = os.path.dirname(__file__) 920 | return bottle.static_file(file, root=os.path.join(current_path, 'assets')) 921 | 922 | 923 | def synch(user_gh, state, repo_label, repo_cfg, repo): 924 | try: 925 | if not repo.is_collaborator(user_gh.user().login): 926 | abort(400, 'You are not a collaborator') 927 | except github3.GitHubError as e: 928 | if e.code == 403: 929 | abort(400, 'Homu does not have write access on the repository') 930 | raise e 931 | 932 | Thread(target=synchronize, args=[repo_label, repo_cfg, g.logger, 933 | g.gh, g.states, g.repos, g.db, 934 | g.mergeable_que, g.my_username, 935 | g.repo_labels]).start() 936 | 937 | return 'Synchronizing {}...'.format(repo_label) 938 | 939 | 940 | def synch_all(): 941 | @retry(wait_exponential_multiplier=1000, wait_exponential_max=600000) 942 | def sync_repo(repo_label, g): 943 | try: 944 | synchronize(repo_label, g.repo_cfgs[repo_label], g.logger, g.gh, 945 | g.states, g.repos, g.db, g.mergeable_que, 946 | g.my_username, g.repo_labels) 947 | except Exception: 948 | print('* Error while synchronizing {}'.format(repo_label)) 949 | traceback.print_exc() 950 | raise 951 | 952 | for repo_label in g.repos: 953 | sync_repo(repo_label, g) 954 | print('* Done synchronizing all') 955 | 956 | 957 | @post('/admin') 958 | def admin(): 959 | if request.json['secret'] != g.cfg['web']['secret']: 960 | return 'Authentication failure' 961 | 962 | if request.json['cmd'] == 'repo_new': 963 | repo_label = request.json['repo_label'] 964 | repo_cfg = request.json['repo_cfg'] 965 | 966 | g.states[repo_label] = {} 967 | g.repos[repo_label] = None 968 | g.repo_cfgs[repo_label] = repo_cfg 969 | g.repo_labels[repo_cfg['owner'], repo_cfg['name']] = repo_label 970 | 971 | Thread(target=synchronize, args=[repo_label, repo_cfg, g.logger, 972 | g.gh, g.states, g.repos, g.db, 973 | g.mergeable_que, g.my_username, 974 | g.repo_labels]).start() 975 | return 'OK' 976 | 977 | elif request.json['cmd'] == 'repo_del': 978 | repo_label = request.json['repo_label'] 979 | repo_cfg = g.repo_cfgs[repo_label] 980 | 981 | db_query(g.db, 'DELETE FROM pull WHERE repo = ?', [repo_label]) 982 | db_query(g.db, 'DELETE FROM build_res WHERE repo = ?', [repo_label]) 983 | db_query(g.db, 'DELETE FROM mergeable WHERE repo = ?', [repo_label]) 984 | 985 | del g.states[repo_label] 986 | del g.repos[repo_label] 987 | del g.repo_cfgs[repo_label] 988 | del g.repo_labels[repo_cfg['owner'], repo_cfg['name']] 989 | 990 | return 'OK' 991 | 992 | elif request.json['cmd'] == 'repo_edit': 993 | repo_label = request.json['repo_label'] 994 | repo_cfg = request.json['repo_cfg'] 995 | 996 | assert repo_cfg['owner'] == g.repo_cfgs[repo_label]['owner'] 997 | assert repo_cfg['name'] == g.repo_cfgs[repo_label]['name'] 998 | 999 | g.repo_cfgs[repo_label] = repo_cfg 1000 | 1001 | return 'OK' 1002 | 1003 | elif request.json['cmd'] == 'sync_all': 1004 | Thread(target=synch_all).start() 1005 | 1006 | return 'OK' 1007 | 1008 | return 'Unrecognized command' 1009 | 1010 | 1011 | @get('/health') 1012 | def health(): 1013 | return 'OK' 1014 | 1015 | 1016 | @error(404) 1017 | def not_found(error): 1018 | return g.tpls['404'].render() 1019 | 1020 | 1021 | def redirect_to_canonical_host(): 1022 | request_url = urllib.parse.urlparse(request.url) 1023 | redirect_url = request_url 1024 | 1025 | # Assume that we're always deployed behind something that hides https:// 1026 | # from us. In production TLS is terminated at ELB, so the actual bors app 1027 | # sees only http:// requests. 1028 | request_url = redirect_url._replace( 1029 | scheme="https" 1030 | ) 1031 | 1032 | # Disable redirects on the health check endpoint. 1033 | if request_url.path == "/health": 1034 | return 1035 | 1036 | # Handle hostname changes 1037 | if "canonical_url" in g.cfg["web"]: 1038 | canonical_url = urllib.parse.urlparse(g.cfg["web"]["canonical_url"]) 1039 | redirect_url = redirect_url._replace( 1040 | scheme=canonical_url.scheme, 1041 | netloc=canonical_url.netloc, 1042 | ) 1043 | 1044 | # Handle path changes 1045 | for prefix in g.cfg["web"].get("remove_path_prefixes", []): 1046 | if redirect_url.path.startswith("/" + prefix + "/"): 1047 | new_path = redirect_url.path[len(prefix)+1:] 1048 | redirect_url = redirect_url._replace(path=new_path) 1049 | elif redirect_url.path == "/" + prefix: 1050 | redirect_url = redirect_url._replace(path="/") 1051 | 1052 | if request_url != redirect_url: 1053 | print("redirecting original=" + request_url.geturl() + " to new=" + redirect_url.geturl()) # noqa 1054 | redirect(urllib.parse.urlunparse(redirect_url), 301) 1055 | 1056 | 1057 | def start(cfg, states, queue_handler, repo_cfgs, repos, logger, 1058 | buildbot_slots, my_username, db, repo_labels, mergeable_que, gh): 1059 | env = jinja2.Environment( 1060 | loader=jinja2.FileSystemLoader(pkg_resources.resource_filename(__name__, 'html')), # noqa 1061 | autoescape=True, 1062 | ) 1063 | env.globals["announcement"] = cfg["web"].get("announcement") 1064 | tpls = {} 1065 | tpls['index'] = env.get_template('index.html') 1066 | tpls['queue'] = env.get_template('queue.html') 1067 | tpls['build_res'] = env.get_template('build_res.html') 1068 | tpls['retry_log'] = env.get_template('retry_log.html') 1069 | tpls['404'] = env.get_template('404.html') 1070 | 1071 | g.cfg = cfg 1072 | g.states = states 1073 | g.queue_handler = queue_handler 1074 | g.repo_cfgs = repo_cfgs 1075 | g.repos = repos 1076 | g.logger = logger.getChild('server') 1077 | g.buildbot_slots = buildbot_slots 1078 | g.tpls = tpls 1079 | g.my_username = my_username 1080 | g.db = db 1081 | g.repo_labels = repo_labels 1082 | g.mergeable_que = mergeable_que 1083 | g.gh = gh 1084 | 1085 | bottle.app().add_hook("before_request", redirect_to_canonical_host) 1086 | 1087 | # Synchronize all PR data on startup 1088 | if cfg['web'].get('sync_on_start', False): 1089 | Thread(target=synch_all).start() 1090 | 1091 | try: 1092 | run(host=cfg['web'].get('host', '0.0.0.0'), 1093 | port=cfg['web']['port'], 1094 | server='waitress') 1095 | except OSError as e: 1096 | print(e, file=sys.stderr) 1097 | os._exit(1) 1098 | -------------------------------------------------------------------------------- /homu/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import github3 3 | import toml 4 | import json 5 | import re 6 | import functools 7 | from . import comments 8 | from . import utils 9 | from .parse_issue_comment import parse_issue_comment 10 | from .auth import verify as verify_auth 11 | from .utils import lazy_debug 12 | import logging 13 | from threading import Thread, Lock, Timer 14 | import time 15 | import traceback 16 | import sqlite3 17 | import requests 18 | from contextlib import contextmanager 19 | from queue import Queue 20 | import os 21 | import sys 22 | from enum import IntEnum, Enum 23 | import subprocess 24 | from .git_helper import SSH_KEY_FILE 25 | import shlex 26 | import random 27 | import weakref 28 | 29 | STATUS_TO_PRIORITY = { 30 | 'pending': 1, 31 | 'approved': 2, 32 | '': 3, 33 | 'error': 4, 34 | 'failure': 5, 35 | 'success': 6, 36 | } 37 | 38 | INTERRUPTED_BY_HOMU_FMT = 'Interrupted by Homu ({})' 39 | INTERRUPTED_BY_HOMU_RE = re.compile(r'Interrupted by Homu \((.+?)\)') 40 | DEFAULT_TEST_TIMEOUT = 3600 * 10 41 | 42 | VARIABLES_RE = re.compile(r'\${([a-zA-Z_]+)}') 43 | 44 | IGNORE_BLOCK_START = '' 45 | IGNORE_BLOCK_END = '' 46 | IGNORE_BLOCK_RE = re.compile( 47 | r'' 48 | r'.*' 49 | r'', 50 | flags=re.MULTILINE | re.DOTALL | re.IGNORECASE 51 | ) 52 | 53 | global_cfg = {} 54 | 55 | 56 | # Replace @mention with `@mention` to suppress pings in merge commits. 57 | # Note: Don't replace non-mentions like "email@gmail.com". 58 | def suppress_pings(text): 59 | return re.sub(r'\B(@\S+)', r'`\g<1>`', text) # noqa 60 | 61 | 62 | # Replace any text between IGNORE_BLOCK_START and IGNORE_BLOCK_END 63 | # HTML comments with an empty string in merge commits 64 | def suppress_ignore_block(text): 65 | return IGNORE_BLOCK_RE.sub('', text) 66 | 67 | 68 | @contextmanager 69 | def buildbot_sess(repo_cfg): 70 | sess = requests.Session() 71 | 72 | sess.post( 73 | repo_cfg['buildbot']['url'] + '/login', 74 | allow_redirects=False, 75 | data={ 76 | 'username': repo_cfg['buildbot']['username'], 77 | 'passwd': repo_cfg['buildbot']['password'], 78 | }) 79 | 80 | yield sess 81 | 82 | sess.get(repo_cfg['buildbot']['url'] + '/logout', allow_redirects=False) 83 | 84 | 85 | db_query_lock = Lock() 86 | 87 | 88 | def db_query(db, *args): 89 | with db_query_lock: 90 | db.execute(*args) 91 | 92 | 93 | class Repository: 94 | treeclosed = -1 95 | treeclosed_src = None 96 | gh = None 97 | gh_test_on_fork = None 98 | label = None 99 | db = None 100 | 101 | def __init__(self, gh, repo_label, db): 102 | self.gh = gh 103 | self.repo_label = repo_label 104 | self.db = db 105 | db_query( 106 | db, 107 | 'SELECT treeclosed, treeclosed_src FROM repos WHERE repo = ?', 108 | [repo_label] 109 | ) 110 | row = db.fetchone() 111 | if row: 112 | self.treeclosed = row[0] 113 | self.treeclosed_src = row[1] 114 | else: 115 | self.treeclosed = -1 116 | self.treeclosed_src = None 117 | 118 | def update_treeclosed(self, value, src): 119 | self.treeclosed = value 120 | self.treeclosed_src = src 121 | db_query( 122 | self.db, 123 | 'DELETE FROM repos where repo = ?', 124 | [self.repo_label] 125 | ) 126 | if value > 0: 127 | db_query( 128 | self.db, 129 | ''' 130 | INSERT INTO repos (repo, treeclosed, treeclosed_src) 131 | VALUES (?, ?, ?) 132 | ''', 133 | [self.repo_label, value, src] 134 | ) 135 | 136 | def __lt__(self, other): 137 | return self.gh < other.gh 138 | 139 | 140 | class PullReqState: 141 | num = 0 142 | priority = 0 143 | rollup = 0 144 | squash = False 145 | title = '' 146 | body = '' 147 | head_ref = '' 148 | base_ref = '' 149 | assignee = '' 150 | delegate = '' 151 | 152 | def __init__(self, num, head_sha, status, db, repo_label, mergeable_que, 153 | gh, owner, name, label_events, repos, test_on_fork): 154 | self.head_advanced('', use_db=False) 155 | 156 | self.num = num 157 | self.head_sha = head_sha 158 | self.status = status 159 | self.db = db 160 | self.repo_label = repo_label 161 | self.mergeable_que = mergeable_que 162 | self.gh = gh 163 | self.owner = owner 164 | self.name = name 165 | self.repos = repos 166 | self.timeout_timer = None 167 | self.test_started = time.time() 168 | self.label_events = label_events 169 | self.test_on_fork = test_on_fork 170 | 171 | def head_advanced(self, head_sha, *, use_db=True): 172 | self.head_sha = head_sha 173 | self.approved_by = '' 174 | self.status = '' 175 | self.merge_sha = '' 176 | self.build_res = {} 177 | self.try_ = False 178 | self.mergeable = None 179 | 180 | if use_db: 181 | self.set_status('') 182 | self.set_mergeable(None) 183 | self.init_build_res([]) 184 | 185 | def __repr__(self): 186 | fmt = 'PullReqState:{}/{}#{}(approved_by={}, priority={}, status={})' 187 | return fmt.format( 188 | self.owner, 189 | self.name, 190 | self.num, 191 | self.approved_by, 192 | self.priority, 193 | self.status, 194 | ) 195 | 196 | def sort_key(self): 197 | return [ 198 | STATUS_TO_PRIORITY.get(self.get_status(), -1), 199 | 1 if self.mergeable is False else 0, 200 | 0 if self.approved_by else 1, 201 | -self.priority, 202 | # Cap `rollup` below at -1 (the value for iffy), so iffy and never 203 | # are treated the same. 204 | max(self.rollup, -1), 205 | self.num, 206 | ] 207 | 208 | def __lt__(self, other): 209 | return self.sort_key() < other.sort_key() 210 | 211 | def get_issue(self): 212 | issue = getattr(self, 'issue', None) 213 | if not issue: 214 | issue = self.issue = self.get_repo().issue(self.num) 215 | return issue 216 | 217 | def add_comment(self, comment): 218 | if isinstance(comment, comments.Comment): 219 | comment = "%s\n" % ( 220 | comment.render(), comment.jsonify(), 221 | ) 222 | self.get_issue().create_comment(comment) 223 | 224 | def change_labels(self, event): 225 | event = self.label_events.get(event.value, {}) 226 | removes = event.get('remove', []) 227 | adds = event.get('add', []) 228 | unless = event.get('unless', []) 229 | if not removes and not adds: 230 | return 231 | 232 | issue = self.get_issue() 233 | labels = {label.name for label in issue.iter_labels()} 234 | if labels.isdisjoint(unless): 235 | labels.difference_update(removes) 236 | labels.update(adds) 237 | issue.replace_labels(list(labels)) 238 | 239 | def set_status(self, status): 240 | self.status = status 241 | if self.timeout_timer: 242 | self.timeout_timer.cancel() 243 | self.timeout_timer = None 244 | 245 | db_query( 246 | self.db, 247 | 'UPDATE pull SET status = ? WHERE repo = ? AND num = ?', 248 | [self.status, self.repo_label, self.num] 249 | ) 250 | 251 | # FIXME: self.try_ should also be saved in the database 252 | if not self.try_: 253 | db_query( 254 | self.db, 255 | 'UPDATE pull SET merge_sha = ? WHERE repo = ? AND num = ?', 256 | [self.merge_sha, self.repo_label, self.num] 257 | ) 258 | 259 | def get_status(self): 260 | if self.status == '' and self.approved_by: 261 | if self.mergeable is not False: 262 | return 'approved' 263 | return self.status 264 | 265 | def set_mergeable(self, mergeable, *, cause=None, que=True): 266 | if mergeable is not None: 267 | self.mergeable = mergeable 268 | 269 | db_query( 270 | self.db, 271 | 'INSERT OR REPLACE INTO mergeable (repo, num, mergeable) VALUES (?, ?, ?)', # noqa 272 | [self.repo_label, self.num, self.mergeable] 273 | ) 274 | else: 275 | if que: 276 | self.mergeable_que.put([self, cause]) 277 | else: 278 | self.mergeable = None 279 | 280 | db_query( 281 | self.db, 282 | 'DELETE FROM mergeable WHERE repo = ? AND num = ?', 283 | [self.repo_label, self.num] 284 | ) 285 | 286 | def init_build_res(self, builders, *, use_db=True): 287 | self.build_res = {x: { 288 | 'res': None, 289 | 'url': '', 290 | } for x in builders} 291 | 292 | if use_db: 293 | db_query( 294 | self.db, 295 | 'DELETE FROM build_res WHERE repo = ? AND num = ?', 296 | [self.repo_label, self.num] 297 | ) 298 | 299 | def set_build_res(self, builder, res, url): 300 | if builder not in self.build_res: 301 | raise Exception('Invalid builder: {}'.format(builder)) 302 | 303 | self.build_res[builder] = { 304 | 'res': res, 305 | 'url': url, 306 | } 307 | 308 | db_query( 309 | self.db, 310 | 'INSERT OR REPLACE INTO build_res (repo, num, builder, res, url, merge_sha) VALUES (?, ?, ?, ?, ?, ?)', # noqa 311 | [ 312 | self.repo_label, 313 | self.num, 314 | builder, 315 | res, 316 | url, 317 | self.merge_sha, 318 | ]) 319 | 320 | def build_res_summary(self): 321 | return ', '.join('{}: {}'.format(builder, data['res']) 322 | for builder, data in self.build_res.items()) 323 | 324 | def get_repo(self): 325 | repo = self.repos[self.repo_label].gh 326 | if not repo: 327 | repo = self.gh.repository(self.owner, self.name) 328 | self.repos[self.repo_label].gh = repo 329 | 330 | assert repo.owner.login == self.owner 331 | assert repo.name == self.name 332 | return repo 333 | 334 | def get_test_on_fork_repo(self): 335 | if not self.test_on_fork: 336 | return None 337 | 338 | repo = self.repos[self.repo_label].gh_test_on_fork 339 | if not repo: 340 | repo = self.gh.repository( 341 | self.test_on_fork['owner'], 342 | self.test_on_fork['name'], 343 | ) 344 | self.repos[self.repo_label].gh_test_on_fork = repo 345 | 346 | assert repo.owner.login == self.test_on_fork['owner'] 347 | assert repo.name == self.test_on_fork['name'] 348 | return repo 349 | 350 | def save(self): 351 | db_query( 352 | self.db, 353 | 'INSERT OR REPLACE INTO pull (repo, num, status, merge_sha, title, body, head_sha, head_ref, base_ref, assignee, approved_by, priority, try_, rollup, squash, delegate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', # noqa 354 | [ 355 | self.repo_label, 356 | self.num, 357 | self.status, 358 | self.merge_sha, 359 | self.title, 360 | self.body, 361 | self.head_sha, 362 | self.head_ref, 363 | self.base_ref, 364 | self.assignee, 365 | self.approved_by, 366 | self.priority, 367 | self.try_, 368 | self.rollup, 369 | self.squash, 370 | self.delegate, 371 | ]) 372 | 373 | def refresh(self): 374 | issue = self.get_repo().issue(self.num) 375 | 376 | self.title = issue.title 377 | self.body = suppress_pings(issue.body or "") 378 | self.body = suppress_ignore_block(self.body) 379 | 380 | def fake_merge(self, repo_cfg): 381 | if not repo_cfg.get('linear', False): 382 | return 383 | if repo_cfg.get('autosquash', False): 384 | return 385 | 386 | issue = self.get_issue() 387 | title = issue.title 388 | # We tell github to close the PR via the commit message, but it 389 | # doesn't know that constitutes a merge. Edit the title so that it's 390 | # clearer. 391 | merged_prefix = '[merged] ' 392 | if not title.startswith(merged_prefix): 393 | title = merged_prefix + title 394 | issue.edit(title=title) 395 | 396 | def change_treeclosed(self, value, src): 397 | self.repos[self.repo_label].update_treeclosed(value, src) 398 | 399 | def blocked_by_closed_tree(self): 400 | treeclosed = self.repos[self.repo_label].treeclosed 401 | return (treeclosed if self.priority < treeclosed else None, 402 | self.repos[self.repo_label].treeclosed_src) 403 | 404 | def start_testing(self, timeout): 405 | self.test_started = time.time() # FIXME: Save in the local database 406 | self.set_status('pending') 407 | 408 | wm = weakref.WeakMethod(self.timed_out) 409 | 410 | def timed_out(): 411 | m = wm() 412 | if m: 413 | m() 414 | timer = Timer(timeout, timed_out) 415 | timer.start() 416 | self.timeout_timer = timer 417 | 418 | def timed_out(self): 419 | print('* Test timed out: {}'.format(self)) 420 | 421 | self.merge_sha = '' 422 | self.save() 423 | self.set_status('failure') 424 | 425 | utils.github_create_status( 426 | self.get_repo(), 427 | self.head_sha, 428 | 'failure', 429 | '', 430 | 'Test timed out', 431 | context='homu') 432 | self.add_comment(comments.TimedOut()) 433 | self.change_labels(LabelEvent.TIMED_OUT) 434 | 435 | def record_retry_log(self, src, body): 436 | # destroy ancient records 437 | db_query( 438 | self.db, 439 | "DELETE FROM retry_log WHERE repo = ? AND time < date('now', ?)", 440 | [self.repo_label, global_cfg.get('retry_log_expire', '-42 days')], 441 | ) 442 | db_query( 443 | self.db, 444 | 'INSERT INTO retry_log (repo, num, src, msg) VALUES (?, ?, ?, ?)', 445 | [self.repo_label, self.num, src, body], 446 | ) 447 | 448 | @property 449 | def author(self): 450 | """ 451 | Get the GitHub login name of the author of the pull request 452 | """ 453 | return self.get_issue().user.login 454 | 455 | 456 | def sha_cmp(short, full): 457 | return len(short) >= 4 and short == full[:len(short)] 458 | 459 | 460 | def sha_or_blank(sha): 461 | return sha if re.match(r'^[0-9a-f]+$', sha) else '' 462 | 463 | 464 | class AuthState(IntEnum): 465 | # Higher is more privileged 466 | REVIEWER = 3 467 | TRY = 2 468 | NONE = 1 469 | 470 | 471 | class LabelEvent(Enum): 472 | APPROVED = 'approved' 473 | REJECTED = 'rejected' 474 | CONFLICT = 'conflict' 475 | SUCCEED = 'succeed' 476 | FAILED = 'failed' 477 | TRY = 'try' 478 | TRY_SUCCEED = 'try_succeed' 479 | TRY_FAILED = 'try_failed' 480 | EXEMPTED = 'exempted' 481 | TIMED_OUT = 'timed_out' 482 | INTERRUPTED = 'interrupted' 483 | PUSHED = 'pushed' 484 | 485 | 486 | PORTAL_TURRET_DIALOG = ["Target acquired", "Activated", "There you are"] 487 | PORTAL_TURRET_IMAGE = "https://cloud.githubusercontent.com/assets/1617736/22222924/c07b2a1c-e16d-11e6-91b3-ac659550585c.png" # noqa 488 | 489 | 490 | def parse_commands(body, username, user_id, repo_label, repo_cfg, state, 491 | my_username, db, states, *, realtime=False, sha='', 492 | command_src=''): 493 | global global_cfg 494 | state_changed = False 495 | 496 | _reviewer_auth_verified = functools.partial( 497 | verify_auth, 498 | username, 499 | user_id, 500 | repo_label, 501 | repo_cfg, 502 | state, 503 | AuthState.REVIEWER, 504 | realtime, 505 | my_username, 506 | ) 507 | _try_auth_verified = functools.partial( 508 | verify_auth, 509 | username, 510 | user_id, 511 | repo_label, 512 | repo_cfg, 513 | state, 514 | AuthState.TRY, 515 | realtime, 516 | my_username, 517 | ) 518 | 519 | hooks = [] 520 | if 'hooks' in global_cfg: 521 | hooks = list(global_cfg['hooks'].keys()) 522 | 523 | commands = parse_issue_comment(username, body, sha, my_username, hooks) 524 | 525 | for command in commands: 526 | found = True 527 | if command.action == 'approve': 528 | if not _reviewer_auth_verified(): 529 | continue 530 | 531 | approver = command.actor 532 | cur_sha = command.commit 533 | 534 | # Ignore WIP PRs 535 | is_wip = False 536 | for wip_kw in ['WIP', 'TODO', '[WIP]', '[TODO]', '[DO NOT MERGE]']: 537 | if state.title.upper().startswith(wip_kw): 538 | if realtime: 539 | state.add_comment(comments.ApprovalIgnoredWip( 540 | sha=state.head_sha, 541 | wip_keyword=wip_kw, 542 | )) 543 | is_wip = True 544 | break 545 | if is_wip: 546 | continue 547 | 548 | # Sometimes, GitHub sends the head SHA of a PR as 0000000 549 | # through the webhook. This is called a "null commit", and 550 | # seems to happen when GitHub internally encounters a race 551 | # condition. Last time, it happened when squashing commits 552 | # in a PR. In this case, we just try to retrieve the head 553 | # SHA manually. 554 | if all(x == '0' for x in state.head_sha): 555 | if realtime: 556 | state.add_comment( 557 | ':bangbang: Invalid head SHA found, retrying: `{}`' 558 | .format(state.head_sha) 559 | ) 560 | 561 | state.head_sha = state.get_repo().pull_request(state.num).head.sha # noqa 562 | state.save() 563 | 564 | assert any(x != '0' for x in state.head_sha) 565 | 566 | if state.approved_by and realtime and username != my_username: 567 | for _state in states[state.repo_label].values(): 568 | if _state.status == 'pending': 569 | break 570 | else: 571 | _state = None 572 | 573 | lines = [] 574 | 575 | if state.status in ['failure', 'error']: 576 | lines.append('- This pull request previously failed. You should add more commits to fix the bug, or use `retry` to trigger a build again.') # noqa 577 | 578 | if _state: 579 | if state == _state: 580 | lines.append('- This pull request is currently being tested. If there\'s no response from the continuous integration service, you may use `retry` to trigger a build again.') # noqa 581 | else: 582 | lines.append('- There\'s another pull request that is currently being tested, blocking this pull request: #{}'.format(_state.num)) # noqa 583 | 584 | if lines: 585 | lines.insert(0, '') 586 | lines.insert(0, ':bulb: This pull request was already approved, no need to approve it again.') # noqa 587 | 588 | state.add_comment('\n'.join(lines)) 589 | 590 | if sha_cmp(cur_sha, state.head_sha): 591 | state.approved_by = approver 592 | state.try_ = False 593 | state.set_status('') 594 | 595 | state.save() 596 | elif realtime and username != my_username: 597 | if cur_sha: 598 | msg = '`{}` is not a valid commit SHA.'.format(cur_sha) 599 | state.add_comment( 600 | ':scream_cat: {} Please try again with `{}`.' 601 | .format(msg, state.head_sha) 602 | ) 603 | else: 604 | state.add_comment(comments.Approved( 605 | sha=state.head_sha, 606 | approver=approver, 607 | bot=my_username, 608 | queue="https://bors.rust-lang.org/homu/queue/{}" 609 | .format(repo_label) 610 | )) 611 | treeclosed, treeclosed_src = state.blocked_by_closed_tree() 612 | if treeclosed: 613 | state.add_comment( 614 | ':evergreen_tree: The tree is currently [closed]({}) for pull requests below priority {}. This pull request will be tested once the tree is reopened.' # noqa 615 | .format(treeclosed_src, treeclosed) 616 | ) 617 | state.change_labels(LabelEvent.APPROVED) 618 | 619 | elif command.action == 'unapprove': 620 | # Allow the author of a pull request to unapprove their own PR. The 621 | # author can already perform other actions that effectively 622 | # unapprove the PR (change the target branch, push more commits, 623 | # etc.) so allowing them to directly unapprove it is also allowed. 624 | 625 | # Because verify_auth has side-effects (especially, it may leave a 626 | # comment on the pull request if the user is not authorized), we 627 | # need to do the author check BEFORE the verify_auth check. 628 | if state.author != username: 629 | if not verify_auth(username, user_id, repo_label, repo_cfg, 630 | state, AuthState.REVIEWER, realtime, 631 | my_username): 632 | continue 633 | 634 | state.approved_by = '' 635 | state.save() 636 | if realtime: 637 | state.change_labels(LabelEvent.REJECTED) 638 | 639 | elif command.action == 'prioritize': 640 | if not verify_auth(username, user_id, repo_label, repo_cfg, state, 641 | AuthState.TRY, realtime, my_username): 642 | continue 643 | 644 | pvalue = command.priority 645 | 646 | if pvalue > global_cfg['max_priority']: 647 | if realtime: 648 | state.add_comment( 649 | ':stop_sign: Priority higher than {} is ignored.' 650 | .format(global_cfg['max_priority']) 651 | ) 652 | continue 653 | state.priority = pvalue 654 | state.save() 655 | 656 | elif command.action == 'delegate': 657 | if not verify_auth(username, user_id, repo_label, repo_cfg, state, 658 | AuthState.REVIEWER, realtime, my_username): 659 | continue 660 | 661 | state.delegate = command.delegate_to 662 | state.save() 663 | 664 | if realtime: 665 | state.add_comment(comments.Delegated( 666 | delegator=username, 667 | delegate=state.delegate, 668 | bot=my_username 669 | )) 670 | 671 | elif command.action == 'undelegate': 672 | # TODO: why is this a TRY? 673 | if not _try_auth_verified(): 674 | continue 675 | state.delegate = '' 676 | state.save() 677 | 678 | elif command.action == 'delegate-author': 679 | if not _reviewer_auth_verified(): 680 | continue 681 | 682 | state.delegate = state.get_repo().pull_request(state.num).user.login # noqa 683 | state.save() 684 | 685 | if realtime: 686 | state.add_comment(comments.Delegated( 687 | delegator=username, 688 | delegate=state.delegate, 689 | bot=my_username 690 | )) 691 | 692 | elif command.action == 'retry' and realtime: 693 | if not _try_auth_verified(): 694 | continue 695 | state.set_status('') 696 | if realtime: 697 | event = LabelEvent.TRY if state.try_ else LabelEvent.APPROVED 698 | state.record_retry_log(command_src, body) 699 | state.change_labels(event) 700 | 701 | elif command.action in ['try', 'untry'] and realtime: 702 | if not _try_auth_verified(): 703 | continue 704 | if state.status == '' and state.approved_by: 705 | state.add_comment( 706 | ':no_good: ' 707 | 'Please do not `try` after a pull request has been `r+`ed.' 708 | ' If you need to `try`, unapprove (`r-`) it first.' 709 | ) 710 | continue 711 | 712 | state.try_ = command.action == 'try' 713 | 714 | state.merge_sha = '' 715 | state.init_build_res([]) 716 | 717 | state.save() 718 | if realtime and state.try_: 719 | # If we've tried before, the status will be 'success', and this 720 | # new try will not be picked up. Set the status back to '' 721 | # so the try will be run again. 722 | state.set_status('') 723 | # `try-` just resets the `try` bit and doesn't correspond to 724 | # any meaningful labeling events. 725 | state.change_labels(LabelEvent.TRY) 726 | 727 | elif command.action == 'rollup': 728 | if not _try_auth_verified(): 729 | continue 730 | state.rollup = command.rollup_value 731 | 732 | state.save() 733 | 734 | elif command.action == 'squash': 735 | if not _try_auth_verified(): 736 | continue 737 | state.squash = True 738 | 739 | state.save() 740 | 741 | elif command.action == 'unsquash': 742 | if not _try_auth_verified(): 743 | continue 744 | state.squash = False 745 | 746 | state.save() 747 | 748 | elif command.action == 'force' and realtime: 749 | if not _try_auth_verified(): 750 | continue 751 | if 'buildbot' in repo_cfg: 752 | with buildbot_sess(repo_cfg) as sess: 753 | res = sess.post( 754 | repo_cfg['buildbot']['url'] + '/builders/_selected/stopselected', # noqa 755 | allow_redirects=False, 756 | data={ 757 | 'selected': repo_cfg['buildbot']['builders'], 758 | 'comments': INTERRUPTED_BY_HOMU_FMT.format(int(time.time())), # noqa 759 | } 760 | ) 761 | 762 | if 'authzfail' in res.text: 763 | err = 'Authorization failed' 764 | else: 765 | mat = re.search('(?s)
(.*?)
', res.text) 766 | if mat: 767 | err = mat.group(1).strip() 768 | if not err: 769 | err = 'Unknown error' 770 | else: 771 | err = '' 772 | 773 | if err: 774 | state.add_comment( 775 | ':bomb: Buildbot returned an error: `{}`'.format(err) 776 | ) 777 | 778 | elif command.action == 'clean' and realtime: 779 | if not _try_auth_verified(): 780 | continue 781 | state.merge_sha = '' 782 | state.init_build_res([]) 783 | 784 | state.save() 785 | 786 | elif command.action == 'ping' and realtime: 787 | if command.ping_type == 'portal': 788 | state.add_comment( 789 | ":cake: {}\n\n![]({})".format( 790 | random.choice(PORTAL_TURRET_DIALOG), 791 | PORTAL_TURRET_IMAGE) 792 | ) 793 | else: 794 | state.add_comment(":sleepy: I'm awake I'm awake") 795 | 796 | elif command.action == 'treeclosed': 797 | if not _reviewer_auth_verified(): 798 | continue 799 | state.change_treeclosed(command.treeclosed_value, command_src) 800 | state.save() 801 | 802 | elif command.action == 'untreeclosed': 803 | if not _reviewer_auth_verified(): 804 | continue 805 | state.change_treeclosed(-1, None) 806 | state.save() 807 | 808 | elif command.action == 'hook': 809 | hook = command.hook_name 810 | hook_cfg = global_cfg['hooks'][hook] 811 | if hook_cfg['realtime'] and not realtime: 812 | continue 813 | if hook_cfg['access'] == "reviewer": 814 | if not _reviewer_auth_verified(): 815 | continue 816 | else: 817 | if not _try_auth_verified(): 818 | continue 819 | Thread( 820 | target=handle_hook_response, 821 | args=[state, hook_cfg, body, command.hook_extra] 822 | ).start() 823 | 824 | else: 825 | found = False 826 | 827 | if found: 828 | state_changed = True 829 | 830 | return state_changed 831 | 832 | 833 | def handle_hook_response(state, hook_cfg, body, extra_data): 834 | post_data = {} 835 | post_data["pull"] = state.num 836 | post_data["body"] = body 837 | post_data["extra_data"] = extra_data 838 | print(post_data) 839 | response = requests.post(hook_cfg['endpoint'], json=post_data) 840 | print(response.text) 841 | 842 | # We only post a response if we're configured to have a response 843 | # non-realtime hooks cannot post 844 | if hook_cfg['has_response'] and hook_cfg['realtime']: 845 | state.add_comment(response.text) 846 | 847 | 848 | def git_push(git_cmd, branch, state): 849 | merge_sha = subprocess.check_output(git_cmd('rev-parse', 'HEAD')).decode('ascii').strip() # noqa 850 | 851 | if utils.silent_call(git_cmd('push', '-f', 'test-origin', branch)): 852 | utils.logged_call(git_cmd('branch', '-f', 'homu-tmp', branch)) 853 | utils.logged_call(git_cmd('push', '-f', 'test-origin', 'homu-tmp')) 854 | 855 | def inner(): 856 | utils.github_create_status( 857 | state.get_repo(), 858 | merge_sha, 859 | 'success', 860 | '', 861 | 'Branch protection bypassed', 862 | context='homu', 863 | ) 864 | 865 | def fail(err): 866 | state.add_comment( 867 | ':boom: Unable to create a status for {} ({})' 868 | .format(merge_sha, err) 869 | ) 870 | 871 | utils.retry_until(inner, fail, state) 872 | 873 | utils.logged_call(git_cmd('push', '-f', 'test-origin', branch)) 874 | 875 | return merge_sha 876 | 877 | 878 | def init_local_git_cmds(repo_cfg, git_cfg): 879 | fpath = os.path.join(git_cfg["cache_dir"], repo_cfg['owner'], repo_cfg['name']) # noqa 880 | genurl = lambda cfg: 'git@github.com:{}/{}.git'.format(cfg['owner'], cfg['name']) # noqa 881 | 882 | if not os.path.exists(SSH_KEY_FILE): 883 | os.makedirs(os.path.dirname(SSH_KEY_FILE), exist_ok=True) 884 | with open(SSH_KEY_FILE, 'w') as fp: 885 | fp.write(git_cfg['ssh_key']) 886 | os.chmod(SSH_KEY_FILE, 0o600) 887 | 888 | if not os.path.exists(fpath): 889 | print("initialized local git repository at", fpath) 890 | utils.logged_call(['git', 'init', fpath]) 891 | 892 | remotes = { 893 | 'origin': genurl(repo_cfg), 894 | 'test-origin': genurl(repo_cfg.get('test-on-fork', repo_cfg)), 895 | } 896 | 897 | for remote, url in remotes.items(): 898 | try: 899 | utils.logged_call(['git', '-C', fpath, 'remote', 'set-url', remote, url]) # noqa 900 | utils.logged_call(['git', '-C', fpath, 'remote', 'set-url', '--push', remote, url]) # noqa 901 | except subprocess.CalledProcessError: 902 | utils.logged_call(['git', '-C', fpath, 'remote', 'add', remote, url]) # noqa 903 | 904 | return lambda *args: ['git', '-C', fpath] + list(args) 905 | 906 | 907 | def branch_equal_to_merge(git_cmd, state, branch): 908 | utils.logged_call(git_cmd('fetch', 'origin', 909 | 'pull/{}/merge'.format(state.num))) 910 | return utils.silent_call(git_cmd('diff', '--quiet', 'FETCH_HEAD', branch)) == 0 # noqa 911 | 912 | 913 | def create_merge(state, repo_cfg, branch, logger, git_cfg, 914 | ensure_merge_equal=False): 915 | # Add some delay to try to make sure the base repo fetch is accurate. 916 | # It seems like in some cases we're getting the previous commit from GH, 917 | # e.g., https://github.com/rust-lang/homu/issues/75#issuecomment-1729058969 918 | # Hopefully a delay helps. 919 | time.sleep(60) 920 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 921 | 922 | state.refresh() 923 | 924 | lazy_debug(logger, 925 | lambda: "create_merge: attempting merge {} into {} on {!r}" 926 | .format(state.head_sha, branch, state.get_repo())) 927 | 928 | merge_msg = 'Auto merge of #{} - {}, r={}\n\n{}\n\n{}'.format( 929 | state.num, 930 | state.head_ref, 931 | '' if state.try_ else state.approved_by, 932 | state.title, 933 | state.body) 934 | 935 | squash_msg = '{}\n\n{}'.format( 936 | state.title, 937 | state.body) 938 | 939 | desc = 'Merge conflict' 940 | comment = ( 941 | 'This pull request and the default branch diverged in ' 942 | 'a way that cannot' 943 | ' be automatically merged. Please rebase on top of the latest default' 944 | ' branch, and let the reviewer approve again.\n' 945 | '\n' 946 | '
How do I rebase?\n\n' 947 | 'Assuming `self` is your fork and `upstream` is this repository,' 948 | ' you can resolve the conflict following these steps:\n\n' 949 | '1. `git checkout {branch}` *(switch to your branch)*\n' 950 | '2. `git fetch upstream HEAD` *(retrieve the latest HEAD)*\n' 951 | '3. `git rebase upstream/HEAD -p` *(rebase on top of it)*\n' 952 | '4. Follow the on-screen instruction to resolve conflicts' 953 | ' (check `git status` if you got lost).\n' 954 | '5. `git push self {branch} --force-with-lease` *(update this PR)*\n\n' 955 | 'You may also read' 956 | ' [*Git Rebasing to Resolve Conflicts* by Drew Blessing](http://blessing.io/git/git-rebase/open-source/2015/08/23/git-rebasing-to-resolve-conflicts.html)' # noqa 957 | ' for a short tutorial.\n\n' 958 | 'Please avoid the ["**Resolve conflicts**" button](https://help.github.com/articles/resolving-a-merge-conflict-on-github/) on GitHub.' # noqa 959 | ' It uses `git merge` instead of `git rebase` which makes the PR commit' # noqa 960 | ' history more difficult to read.\n\n' 961 | 'Sometimes step 4 will complete without asking for resolution. This is' 962 | ' usually due to difference between how `Cargo.lock` conflict is' 963 | ' handled during merge and rebase. This is normal, and you should still' # noqa 964 | ' perform step 5 to update this PR.\n\n' 965 | '
\n\n' 966 | ).format(branch=state.head_ref.split(':', 1)[1]) 967 | 968 | if git_cfg['local_git']: 969 | 970 | git_cmd = init_local_git_cmds(repo_cfg, git_cfg) 971 | 972 | utils.logged_call(git_cmd('fetch', 'origin', state.base_ref, 973 | 'pull/{}/head'.format(state.num))) 974 | utils.silent_call(git_cmd('reset', '--hard')) 975 | utils.silent_call(git_cmd('rebase', '--abort')) 976 | utils.silent_call(git_cmd('merge', '--abort')) 977 | 978 | if repo_cfg.get('linear', False): 979 | utils.logged_call( 980 | git_cmd('checkout', '-B', branch, state.head_sha)) 981 | try: 982 | args = [base_sha] 983 | if repo_cfg.get('autosquash', False): 984 | args += ['-i', '--autosquash'] 985 | utils.logged_call(git_cmd('-c', 986 | 'user.name=' + git_cfg['name'], 987 | '-c', 988 | 'user.email=' + git_cfg['email'], 989 | 'rebase', 990 | *args)) 991 | except subprocess.CalledProcessError: 992 | if repo_cfg.get('autosquash', False): 993 | utils.silent_call(git_cmd('rebase', '--abort')) 994 | if utils.silent_call(git_cmd('rebase', base_sha)) == 0: 995 | desc = 'Auto-squashing failed' 996 | comment = '' 997 | else: 998 | ap = '' if state.try_ else state.approved_by 999 | text = '\nCloses: #{}\nApproved by: {}'.format(state.num, ap) 1000 | msg_code = 'cat && echo {}'.format(shlex.quote(text)) 1001 | env_code = 'export GIT_COMMITTER_NAME={} && export GIT_COMMITTER_EMAIL={} && unset GIT_COMMITTER_DATE'.format(shlex.quote(git_cfg['name']), shlex.quote(git_cfg['email'])) # noqa 1002 | utils.logged_call(git_cmd('filter-branch', '-f', 1003 | '--msg-filter', msg_code, 1004 | '--env-filter', env_code, 1005 | '{}..'.format(base_sha))) 1006 | 1007 | if ensure_merge_equal: 1008 | if not branch_equal_to_merge(git_cmd, state, branch): 1009 | return '' 1010 | 1011 | return git_push(git_cmd, branch, state) 1012 | else: 1013 | utils.logged_call(git_cmd( 1014 | 'checkout', 1015 | '-f', 1016 | '-B', 1017 | 'homu-tmp', 1018 | state.head_sha)) 1019 | 1020 | ok = True 1021 | if repo_cfg.get('autosquash', False): 1022 | try: 1023 | merge_base_sha = subprocess.check_output( 1024 | git_cmd( 1025 | 'merge-base', 1026 | base_sha, 1027 | state.head_sha)).decode('ascii').strip() 1028 | utils.logged_call(git_cmd( 1029 | '-c', 1030 | 'user.name=' + git_cfg['name'], 1031 | '-c', 1032 | 'user.email=' + git_cfg['email'], 1033 | 'rebase', 1034 | '-i', 1035 | '--autosquash', 1036 | '--onto', 1037 | merge_base_sha, base_sha)) 1038 | except subprocess.CalledProcessError: 1039 | desc = 'Auto-squashing failed' 1040 | comment = '' 1041 | ok = False 1042 | if state.squash: 1043 | try: 1044 | merge_base_sha = subprocess.check_output( 1045 | git_cmd( 1046 | 'merge-base', 1047 | base_sha, 1048 | state.head_sha)).decode('ascii').strip() 1049 | utils.logged_call(git_cmd( 1050 | 'reset', 1051 | '--soft', 1052 | merge_base_sha)) 1053 | utils.logged_call(git_cmd( 1054 | '-c', 1055 | 'user.name=' + git_cfg['name'], 1056 | '-c', 1057 | 'user.email=' + git_cfg['email'], 1058 | 'commit', 1059 | '-m', 1060 | squash_msg)) 1061 | except subprocess.CalledProcessError: 1062 | desc = 'Squashing failed' 1063 | comment = '' 1064 | ok = False 1065 | 1066 | if ok: 1067 | utils.logged_call(git_cmd('checkout', '-B', branch, base_sha)) 1068 | try: 1069 | subprocess.check_output( 1070 | git_cmd( 1071 | '-c', 1072 | 'user.name=' + git_cfg['name'], 1073 | '-c', 1074 | 'user.email=' + git_cfg['email'], 1075 | 'merge', 1076 | 'heads/homu-tmp', 1077 | '--no-ff', 1078 | '-m', 1079 | merge_msg), 1080 | stderr=subprocess.STDOUT, 1081 | universal_newlines=True) 1082 | except subprocess.CalledProcessError as e: 1083 | comment += '
Error message\n\n```text\n' # noqa 1084 | comment += e.output 1085 | comment += '\n```\n\n
' 1086 | pass 1087 | else: 1088 | if ensure_merge_equal: 1089 | if not branch_equal_to_merge(git_cmd, state, branch): 1090 | return '' 1091 | 1092 | return git_push(git_cmd, branch, state) 1093 | else: 1094 | if repo_cfg.get('linear', False) or repo_cfg.get('autosquash', False): 1095 | raise RuntimeError('local_git must be turned on to use this feature') # noqa 1096 | 1097 | # if we're merging using the GitHub API, we have no way to predict 1098 | # with certainty what the final result will be so make sure the caller 1099 | # isn't asking us to keep any promises (see also discussions at 1100 | # https://github.com/servo/homu/pull/57) 1101 | assert ensure_merge_equal is False 1102 | 1103 | if branch != state.base_ref: 1104 | utils.github_set_ref( 1105 | state.get_repo(), 1106 | 'heads/' + branch, 1107 | base_sha, 1108 | force=True, 1109 | ) 1110 | 1111 | try: 1112 | merge_commit = state.get_repo().merge( 1113 | branch, 1114 | state.head_sha, 1115 | merge_msg) 1116 | except github3.models.GitHubError as e: 1117 | if e.code != 409: 1118 | raise 1119 | else: 1120 | return merge_commit.sha if merge_commit else '' 1121 | 1122 | state.set_status('error') 1123 | utils.github_create_status( 1124 | state.get_repo(), 1125 | state.head_sha, 1126 | 'error', 1127 | '', 1128 | desc, 1129 | context='homu') 1130 | 1131 | state.add_comment(':lock: {}\n\n{}'.format(desc, comment)) 1132 | state.change_labels(LabelEvent.CONFLICT) 1133 | 1134 | return '' 1135 | 1136 | 1137 | def pull_is_rebased(state, repo_cfg, git_cfg, base_sha): 1138 | assert git_cfg['local_git'] 1139 | git_cmd = init_local_git_cmds(repo_cfg, git_cfg) 1140 | 1141 | utils.logged_call(git_cmd('fetch', 'origin', state.base_ref, 1142 | 'pull/{}/head'.format(state.num))) 1143 | 1144 | return utils.silent_call(git_cmd('merge-base', '--is-ancestor', 1145 | base_sha, state.head_sha)) == 0 1146 | 1147 | 1148 | # We could fetch this from GitHub instead, but that API is being deprecated: 1149 | # https://developer.github.com/changes/2013-04-25-deprecating-merge-commit-sha/ 1150 | def get_github_merge_sha(state, repo_cfg, git_cfg): 1151 | assert git_cfg['local_git'] 1152 | git_cmd = init_local_git_cmds(repo_cfg, git_cfg) 1153 | 1154 | if state.mergeable is not True: 1155 | return None 1156 | 1157 | utils.logged_call(git_cmd('fetch', 'origin', 1158 | 'pull/{}/merge'.format(state.num))) 1159 | 1160 | return subprocess.check_output(git_cmd('rev-parse', 'FETCH_HEAD')).decode('ascii').strip() # noqa 1161 | 1162 | 1163 | def do_exemption_merge(state, logger, repo_cfg, git_cfg, url, check_merge, 1164 | reason): 1165 | 1166 | try: 1167 | merge_sha = create_merge( 1168 | state, 1169 | repo_cfg, 1170 | state.base_ref, 1171 | logger, 1172 | git_cfg, 1173 | check_merge) 1174 | except subprocess.CalledProcessError: 1175 | print('* Unable to create a merge commit for the exempted PR: {}'.format(state)) # noqa 1176 | traceback.print_exc() 1177 | return False 1178 | 1179 | if not merge_sha: 1180 | return False 1181 | 1182 | desc = 'Test exempted' 1183 | 1184 | state.set_status('success') 1185 | utils.github_create_status(state.get_repo(), state.head_sha, 'success', 1186 | url, desc, context='homu') 1187 | state.add_comment(':zap: {}: {}.'.format(desc, reason)) 1188 | state.change_labels(LabelEvent.EXEMPTED) 1189 | 1190 | state.merge_sha = merge_sha 1191 | state.save() 1192 | 1193 | state.fake_merge(repo_cfg) 1194 | return True 1195 | 1196 | 1197 | def try_travis_exemption(state, logger, repo_cfg, git_cfg): 1198 | 1199 | travis_info = None 1200 | for info in utils.github_iter_statuses(state.get_repo(), state.head_sha): 1201 | if info.context == 'continuous-integration/travis-ci/pr': 1202 | travis_info = info 1203 | break 1204 | 1205 | if travis_info is None or travis_info.state != 'success': 1206 | return False 1207 | 1208 | mat = re.search('/builds/([0-9]+)$', travis_info.target_url) 1209 | if not mat: 1210 | return False 1211 | 1212 | url = 'https://api.travis-ci.org/{}/{}/builds/{}'.format(state.owner, 1213 | state.name, 1214 | mat.group(1)) 1215 | try: 1216 | res = requests.get(url) 1217 | except Exception as ex: 1218 | print('* Unable to gather build info from Travis CI: {}'.format(ex)) 1219 | return False 1220 | 1221 | travis_sha = json.loads(res.text)['commit'] 1222 | travis_commit = state.get_repo().commit(travis_sha) 1223 | 1224 | if not travis_commit: 1225 | return False 1226 | 1227 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 1228 | 1229 | if (travis_commit.parents[0]['sha'] == base_sha and 1230 | travis_commit.parents[1]['sha'] == state.head_sha): 1231 | # make sure we check against the github merge sha before pushing 1232 | return do_exemption_merge(state, logger, repo_cfg, git_cfg, 1233 | travis_info.target_url, True, 1234 | "merge already tested by Travis CI") 1235 | 1236 | return False 1237 | 1238 | 1239 | def try_status_exemption(state, logger, repo_cfg, git_cfg): 1240 | 1241 | # If all the builders are status-based, then we can do some checks to 1242 | # exempt testing under the following cases: 1243 | # 1. The PR head commit has the equivalent statuses set to 'success' and 1244 | # it is fully rebased on the HEAD of the target base ref. 1245 | # 2. The PR head and merge commits have the equivalent statuses set to 1246 | # state 'success' and the merge commit's first parent is the HEAD of 1247 | # the target base ref. 1248 | 1249 | if not git_cfg['local_git']: 1250 | raise RuntimeError('local_git is required to use status exemption') 1251 | 1252 | statuses_all = set() 1253 | 1254 | # equivalence dict: pr context --> auto context 1255 | status_equivalences = {} 1256 | 1257 | for key, value in repo_cfg['status'].items(): 1258 | context = value.get('context') 1259 | pr_context = value.get('pr_context', context) 1260 | if context is not None: 1261 | statuses_all.add(context) 1262 | status_equivalences[pr_context] = context 1263 | 1264 | assert len(statuses_all) > 0 1265 | 1266 | # let's first check that all the statuses we want are set to success 1267 | statuses_pass = set() 1268 | for info in utils.github_iter_statuses(state.get_repo(), state.head_sha): 1269 | if info.context in status_equivalences and info.state == 'success': 1270 | statuses_pass.add(status_equivalences[info.context]) 1271 | 1272 | if statuses_all != statuses_pass: 1273 | return False 1274 | 1275 | # is the PR fully rebased? 1276 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 1277 | if pull_is_rebased(state, repo_cfg, git_cfg, base_sha): 1278 | return do_exemption_merge(state, logger, repo_cfg, git_cfg, '', False, 1279 | "pull fully rebased and already tested") 1280 | 1281 | # check if we can use the github merge sha as proof 1282 | merge_sha = get_github_merge_sha(state, repo_cfg, git_cfg) 1283 | if merge_sha is None: 1284 | return False 1285 | 1286 | statuses_merge_pass = set() 1287 | for info in utils.github_iter_statuses(state.get_repo(), merge_sha): 1288 | if info.context in status_equivalences and info.state == 'success': 1289 | statuses_merge_pass.add(status_equivalences[info.context]) 1290 | 1291 | merge_commit = state.get_repo().commit(merge_sha) 1292 | if (statuses_all == statuses_merge_pass and 1293 | merge_commit.parents[0]['sha'] == base_sha and 1294 | merge_commit.parents[1]['sha'] == state.head_sha): 1295 | # make sure we check against the github merge sha before pushing 1296 | return do_exemption_merge(state, logger, repo_cfg, git_cfg, '', True, 1297 | "merge already tested") 1298 | 1299 | return False 1300 | 1301 | 1302 | def start_build(state, repo_cfgs, buildbot_slots, logger, db, git_cfg): 1303 | if buildbot_slots[0]: 1304 | return True 1305 | 1306 | lazy_debug(logger, lambda: "start_build on {!r}".format(state.get_repo())) 1307 | 1308 | pr = state.get_repo().pull_request(state.num) 1309 | assert state.head_sha == pr.head.sha 1310 | assert state.base_ref == pr.base.ref 1311 | 1312 | repo_cfg = repo_cfgs[state.repo_label] 1313 | 1314 | builders = [] 1315 | branch = 'try' if state.try_ else 'auto' 1316 | branch = repo_cfg.get('branch', {}).get(branch, branch) 1317 | can_try_travis_exemption = False 1318 | 1319 | only_status_builders = True 1320 | if 'buildbot' in repo_cfg: 1321 | if state.try_: 1322 | builders += repo_cfg['buildbot']['try_builders'] 1323 | else: 1324 | builders += repo_cfg['buildbot']['builders'] 1325 | only_status_builders = False 1326 | if 'travis' in repo_cfg: 1327 | builders += ['travis'] 1328 | only_status_builders = False 1329 | if 'status' in repo_cfg: 1330 | found_travis_context = False 1331 | for key, value in repo_cfg['status'].items(): 1332 | context = value.get('context') 1333 | if context is not None: 1334 | if state.try_ and not value.get('try', True): 1335 | # Skip this builder for tries. 1336 | continue 1337 | builders += ['status-' + key] 1338 | # We have an optional fast path if the Travis test passed 1339 | # for a given commit and main is unchanged, we can do 1340 | # a direct push. 1341 | if context == 'continuous-integration/travis-ci/push': 1342 | found_travis_context = True 1343 | 1344 | if found_travis_context and len(builders) == 1: 1345 | can_try_travis_exemption = True 1346 | if 'checks' in repo_cfg: 1347 | builders += [ 1348 | 'checks-' + key 1349 | for key, value in repo_cfg['checks'].items() 1350 | if 'name' in value or (state.try_ and 'try_name' in value) 1351 | ] 1352 | only_status_builders = False 1353 | 1354 | if len(builders) == 0: 1355 | raise RuntimeError('Invalid configuration') 1356 | 1357 | lazy_debug(logger, lambda: "start_build: builders={!r}".format(builders)) 1358 | 1359 | if (only_status_builders and state.approved_by and 1360 | repo_cfg.get('status_based_exemption', False)): 1361 | if can_try_travis_exemption: 1362 | if try_travis_exemption(state, logger, repo_cfg, git_cfg): 1363 | return True 1364 | if try_status_exemption(state, logger, repo_cfg, git_cfg): 1365 | return True 1366 | 1367 | merge_sha = create_merge(state, repo_cfg, branch, logger, git_cfg) 1368 | lazy_debug(logger, lambda: "start_build: merge_sha={}".format(merge_sha)) 1369 | if not merge_sha: 1370 | return False 1371 | 1372 | state.init_build_res(builders) 1373 | state.merge_sha = merge_sha 1374 | 1375 | state.save() 1376 | 1377 | if 'buildbot' in repo_cfg: 1378 | buildbot_slots[0] = state.merge_sha 1379 | 1380 | logger.info('Starting build of {}/{}#{} on {}: {}'.format( 1381 | state.owner, 1382 | state.name, 1383 | state.num, 1384 | branch, 1385 | state.merge_sha)) 1386 | 1387 | timeout = repo_cfg.get('timeout', DEFAULT_TEST_TIMEOUT) 1388 | state.start_testing(timeout) 1389 | 1390 | desc = '{} commit {} with merge {}...'.format( 1391 | 'Trying' if state.try_ else 'Testing', 1392 | state.head_sha, 1393 | state.merge_sha, 1394 | ) 1395 | utils.github_create_status( 1396 | state.get_repo(), 1397 | state.head_sha, 1398 | 'pending', 1399 | '', 1400 | desc, 1401 | context='homu') 1402 | 1403 | if state.try_: 1404 | state.add_comment(comments.TryBuildStarted( 1405 | head_sha=state.head_sha, 1406 | merge_sha=state.merge_sha, 1407 | )) 1408 | else: 1409 | state.add_comment(comments.BuildStarted( 1410 | head_sha=state.head_sha, 1411 | merge_sha=state.merge_sha, 1412 | )) 1413 | 1414 | return True 1415 | 1416 | 1417 | def start_rebuild(state, repo_cfgs): 1418 | repo_cfg = repo_cfgs[state.repo_label] 1419 | 1420 | if 'buildbot' not in repo_cfg or not state.build_res: 1421 | return False 1422 | 1423 | builders = [] 1424 | succ_builders = [] 1425 | 1426 | for builder, info in state.build_res.items(): 1427 | if not info['url']: 1428 | return False 1429 | 1430 | if info['res']: 1431 | succ_builders.append([builder, info['url']]) 1432 | else: 1433 | builders.append([builder, info['url']]) 1434 | 1435 | if not builders or not succ_builders: 1436 | return False 1437 | 1438 | base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha 1439 | _parents = state.get_repo().commit(state.merge_sha).parents 1440 | parent_shas = [x['sha'] for x in _parents] 1441 | 1442 | if base_sha not in parent_shas: 1443 | return False 1444 | 1445 | utils.github_set_ref( 1446 | state.get_repo(), 1447 | 'tags/homu-tmp', 1448 | state.merge_sha, 1449 | force=True) 1450 | 1451 | builders.sort() 1452 | succ_builders.sort() 1453 | 1454 | with buildbot_sess(repo_cfg) as sess: 1455 | for builder, url in builders: 1456 | res = sess.post(url + '/rebuild', allow_redirects=False, data={ 1457 | 'useSourcestamp': 'exact', 1458 | 'comments': 'Initiated by Homu', 1459 | }) 1460 | 1461 | if 'authzfail' in res.text: 1462 | err = 'Authorization failed' 1463 | elif builder in res.text: 1464 | err = '' 1465 | else: 1466 | mat = re.search('(.+?)', res.text) 1467 | err = mat.group(1) if mat else 'Unknown error' 1468 | 1469 | if err: 1470 | state.add_comment(':bomb: Failed to start rebuilding: `{}`'.format(err)) # noqa 1471 | return False 1472 | 1473 | timeout = repo_cfg.get('timeout', DEFAULT_TEST_TIMEOUT) 1474 | state.start_testing(timeout) 1475 | 1476 | msg_1 = 'Previous build results' 1477 | msg_2 = ' for {}'.format(', '.join('[{}]({})'.format(builder, url) for builder, url in succ_builders)) # noqa 1478 | msg_3 = ' are reusable. Rebuilding' 1479 | msg_4 = ' only {}'.format(', '.join('[{}]({})'.format(builder, url) for builder, url in builders)) # noqa 1480 | 1481 | utils.github_create_status( 1482 | state.get_repo(), 1483 | state.head_sha, 1484 | 'pending', 1485 | '', 1486 | '{}{}...'.format(msg_1, msg_3), 1487 | context='homu') 1488 | 1489 | state.add_comment(':zap: {}{}{}{}...'.format(msg_1, msg_2, msg_3, msg_4)) 1490 | 1491 | return True 1492 | 1493 | 1494 | def start_build_or_rebuild(state, repo_cfgs, *args): 1495 | if start_rebuild(state, repo_cfgs): 1496 | return True 1497 | 1498 | return start_build(state, repo_cfgs, *args) 1499 | 1500 | 1501 | def process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, 1502 | git_cfg): 1503 | for repo_label, repo in repos.items(): 1504 | repo_states = sorted(states[repo_label].values()) 1505 | 1506 | for state in repo_states: 1507 | lazy_debug(logger, lambda: "process_queue: state={!r}, building {}" 1508 | .format(state, repo_label)) 1509 | if state.priority < repo.treeclosed: 1510 | continue 1511 | if state.status == 'pending' and not state.try_: 1512 | break 1513 | 1514 | elif state.status == 'success' and hasattr(state, 'fake_merge_sha'): # noqa 1515 | break 1516 | 1517 | elif state.status == '' and state.approved_by: 1518 | if start_build_or_rebuild(state, repo_cfgs, buildbot_slots, 1519 | logger, db, git_cfg): 1520 | return 1521 | 1522 | elif state.status == 'success' and state.try_ and state.approved_by: # noqa 1523 | state.try_ = False 1524 | 1525 | state.save() 1526 | 1527 | if start_build(state, repo_cfgs, buildbot_slots, logger, db, 1528 | git_cfg): 1529 | return 1530 | 1531 | for state in repo_states: 1532 | if state.status == '' and state.try_: 1533 | if start_build(state, repo_cfgs, buildbot_slots, logger, db, 1534 | git_cfg): 1535 | return 1536 | 1537 | 1538 | def fetch_mergeability(mergeable_que): 1539 | re_pull_num = re.compile('(?i)merge (?:of|pull request) #([0-9]+)') 1540 | 1541 | while True: 1542 | try: 1543 | state, cause = mergeable_que.get() 1544 | 1545 | if state.status == 'success': 1546 | continue 1547 | 1548 | pull_request = state.get_repo().pull_request(state.num) 1549 | if pull_request is None or pull_request.mergeable is None: 1550 | time.sleep(5) 1551 | pull_request = state.get_repo().pull_request(state.num) 1552 | mergeable = pull_request is not None and pull_request.mergeable 1553 | 1554 | if state.mergeable is True and mergeable is False: 1555 | if cause: 1556 | mat = re_pull_num.search(cause['title']) 1557 | 1558 | if mat: 1559 | issue_or_commit = '#' + mat.group(1) 1560 | else: 1561 | issue_or_commit = cause['sha'] 1562 | else: 1563 | issue_or_commit = '' 1564 | 1565 | _blame = '' 1566 | if issue_or_commit: 1567 | _blame = ' (presumably {})'.format(issue_or_commit) 1568 | state.add_comment( 1569 | ':umbrella: The latest upstream changes{} made this ' 1570 | 'pull request unmergeable. Please [resolve the merge conflicts]' # noqa 1571 | '(https://rustc-dev-guide.rust-lang.org/git.html#rebasing-and-conflicts).' # noqa 1572 | .format(_blame) 1573 | ) 1574 | state.change_labels(LabelEvent.CONFLICT) 1575 | 1576 | state.set_mergeable(mergeable, que=False) 1577 | 1578 | except Exception: 1579 | print('* Error while fetching mergeability') 1580 | traceback.print_exc() 1581 | 1582 | finally: 1583 | mergeable_que.task_done() 1584 | 1585 | 1586 | def synchronize(repo_label, repo_cfg, logger, gh, states, repos, db, mergeable_que, my_username, repo_labels): # noqa 1587 | logger.info('Synchronizing {}...'.format(repo_label)) 1588 | 1589 | repo = gh.repository(repo_cfg['owner'], repo_cfg['name']) 1590 | 1591 | db_query(db, 'DELETE FROM pull WHERE repo = ?', [repo_label]) 1592 | db_query(db, 'DELETE FROM build_res WHERE repo = ?', [repo_label]) 1593 | db_query(db, 'DELETE FROM mergeable WHERE repo = ?', [repo_label]) 1594 | 1595 | saved_states = {} 1596 | for num, state in states[repo_label].items(): 1597 | saved_states[num] = { 1598 | 'merge_sha': state.merge_sha, 1599 | 'build_res': state.build_res, 1600 | } 1601 | 1602 | states[repo_label] = {} 1603 | repos[repo_label] = Repository(repo, repo_label, db) 1604 | 1605 | for pull in repo.iter_pulls(state='open'): 1606 | db_query( 1607 | db, 1608 | 'SELECT status FROM pull WHERE repo = ? AND num = ?', 1609 | [repo_label, pull.number]) 1610 | row = db.fetchone() 1611 | if row: 1612 | status = row[0] 1613 | else: 1614 | status = '' 1615 | for info in utils.github_iter_statuses(repo, pull.head.sha): 1616 | if info.context == 'homu': 1617 | status = info.state 1618 | break 1619 | 1620 | state = PullReqState(pull.number, pull.head.sha, status, db, repo_label, mergeable_que, gh, repo_cfg['owner'], repo_cfg['name'], repo_cfg.get('labels', {}), repos, repo_cfg.get('test-on-fork')) # noqa 1621 | state.title = pull.title 1622 | state.body = suppress_pings(pull.body or "") 1623 | state.body = suppress_ignore_block(state.body) 1624 | state.head_ref = pull.head.repo[0] + ':' + pull.head.ref 1625 | state.base_ref = pull.base.ref 1626 | state.set_mergeable(None) 1627 | state.assignee = pull.assignee.login if pull.assignee else '' 1628 | 1629 | for comment in pull.iter_comments(): 1630 | if comment.original_commit_id == pull.head.sha: 1631 | parse_commands( 1632 | comment.body, 1633 | comment.user.login, 1634 | comment.user.id, 1635 | repo_label, 1636 | repo_cfg, 1637 | state, 1638 | my_username, 1639 | db, 1640 | states, 1641 | sha=comment.original_commit_id, 1642 | command_src=comment.to_json()['html_url'], 1643 | # FIXME switch to `comment.html_url` 1644 | # after updating github3 to 1.3.0+ 1645 | ) 1646 | 1647 | for comment in pull.iter_issue_comments(): 1648 | parse_commands( 1649 | comment.body, 1650 | comment.user.login, 1651 | comment.user.id, 1652 | repo_label, 1653 | repo_cfg, 1654 | state, 1655 | my_username, 1656 | db, 1657 | states, 1658 | command_src=comment.to_json()['html_url'], 1659 | # FIXME switch to `comment.html_url` 1660 | # after updating github3 to 1.3.0+ 1661 | ) 1662 | 1663 | saved_state = saved_states.get(pull.number) 1664 | if saved_state: 1665 | for key, val in saved_state.items(): 1666 | setattr(state, key, val) 1667 | 1668 | state.save() 1669 | 1670 | states[repo_label][pull.number] = state 1671 | 1672 | logger.info('Done synchronizing {}!'.format(repo_label)) 1673 | 1674 | 1675 | def process_config(config): 1676 | # Replace environment variables 1677 | if type(config) is str: 1678 | for var in VARIABLES_RE.findall(config): 1679 | try: 1680 | config = config.replace("${"+var+"}", os.environ[var]) 1681 | except KeyError: 1682 | raise RuntimeError( 1683 | f"missing environment variable ${var} " 1684 | f"(requested in the configuration file)" 1685 | ) from None 1686 | 1687 | return config 1688 | # Recursively apply the processing 1689 | elif type(config) is list: 1690 | return [process_config(item) for item in config] 1691 | elif type(config) is dict: 1692 | return {key: process_config(value) for key, value in config.items()} 1693 | # All other values should be returned as-is 1694 | else: 1695 | return config 1696 | 1697 | 1698 | def arguments(): 1699 | parser = argparse.ArgumentParser( 1700 | description='A bot that integrates with GitHub and your favorite ' 1701 | 'continuous integration service') 1702 | parser.add_argument( 1703 | '-v', 1704 | '--verbose', 1705 | action='store_true', 1706 | help='Enable more verbose logging') 1707 | parser.add_argument( 1708 | '-c', 1709 | '--config', 1710 | action='store', 1711 | help='Path to cfg.toml', 1712 | default='cfg.toml') 1713 | 1714 | return parser.parse_args() 1715 | 1716 | 1717 | def main(): 1718 | global global_cfg 1719 | args = arguments() 1720 | 1721 | logger = logging.getLogger('homu') 1722 | logger.setLevel(logging.DEBUG if args.verbose else logging.INFO) 1723 | logger.addHandler(logging.StreamHandler()) 1724 | 1725 | if sys.getfilesystemencoding() == 'ascii': 1726 | logger.info('You need to set a locale compatible with unicode or homu will choke on Unicode in PR descriptions/titles. See http://stackoverflow.com/a/27931669') # noqa 1727 | 1728 | try: 1729 | with open(args.config) as fp: 1730 | cfg = toml.loads(fp.read()) 1731 | except FileNotFoundError: 1732 | # Fall back to cfg.json only if we're using the defaults 1733 | if args.config == 'cfg.toml': 1734 | with open('cfg.json') as fp: 1735 | cfg = json.loads(fp.read()) 1736 | else: 1737 | raise 1738 | cfg = process_config(cfg) 1739 | global_cfg = cfg 1740 | 1741 | gh = github3.login(token=cfg['github']['access_token']) 1742 | user = gh.user() 1743 | cfg_git = cfg.get('git', {}) 1744 | user_email = cfg_git.get('email') 1745 | if user_email is None: 1746 | try: 1747 | user_email = [x for x in gh.iter_emails() if x['primary']][0]['email'] # noqa 1748 | except IndexError: 1749 | raise RuntimeError('Primary email not set, or "user" scope not granted') # noqa 1750 | user_name = cfg_git.get('name', user.name if user.name else user.login) 1751 | 1752 | states = {} 1753 | repos = {} 1754 | repo_cfgs = {} 1755 | buildbot_slots = [''] 1756 | my_username = user.login 1757 | repo_labels = {} 1758 | mergeable_que = Queue() 1759 | git_cfg = { 1760 | 'name': user_name, 1761 | 'email': user_email, 1762 | 'ssh_key': cfg_git.get('ssh_key', ''), 1763 | 'local_git': cfg_git.get('local_git', False), 1764 | 'cache_dir': cfg_git.get('cache_dir', 'cache') 1765 | } 1766 | 1767 | db_file = cfg.get('db', {}).get('file', 'main.db') 1768 | db_conn = sqlite3.connect(db_file, 1769 | check_same_thread=False, 1770 | isolation_level=None) 1771 | db = db_conn.cursor() 1772 | 1773 | db_query(db, '''CREATE TABLE IF NOT EXISTS pull ( 1774 | repo TEXT NOT NULL, 1775 | num INTEGER NOT NULL, 1776 | status TEXT NOT NULL, 1777 | merge_sha TEXT, 1778 | title TEXT, 1779 | body TEXT, 1780 | head_sha TEXT, 1781 | head_ref TEXT, 1782 | base_ref TEXT, 1783 | assignee TEXT, 1784 | approved_by TEXT, 1785 | priority INTEGER, 1786 | try_ INTEGER, 1787 | rollup INTEGER, 1788 | squash INTEGER, 1789 | delegate TEXT, 1790 | UNIQUE (repo, num) 1791 | )''') 1792 | 1793 | db_query(db, '''CREATE TABLE IF NOT EXISTS build_res ( 1794 | repo TEXT NOT NULL, 1795 | num INTEGER NOT NULL, 1796 | builder TEXT NOT NULL, 1797 | res INTEGER, 1798 | url TEXT NOT NULL, 1799 | merge_sha TEXT NOT NULL, 1800 | UNIQUE (repo, num, builder) 1801 | )''') 1802 | 1803 | db_query(db, '''CREATE TABLE IF NOT EXISTS mergeable ( 1804 | repo TEXT NOT NULL, 1805 | num INTEGER NOT NULL, 1806 | mergeable INTEGER NOT NULL, 1807 | UNIQUE (repo, num) 1808 | )''') 1809 | db_query(db, '''CREATE TABLE IF NOT EXISTS repos ( 1810 | repo TEXT NOT NULL, 1811 | treeclosed INTEGER NOT NULL, 1812 | treeclosed_src TEXT, 1813 | UNIQUE (repo) 1814 | )''') 1815 | 1816 | db_query(db, '''CREATE TABLE IF NOT EXISTS retry_log ( 1817 | repo TEXT NOT NULL, 1818 | num INTEGER NOT NULL, 1819 | time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, 1820 | src TEXT NOT NULL, 1821 | msg TEXT NOT NULL 1822 | )''') 1823 | db_query(db, ''' 1824 | CREATE INDEX IF NOT EXISTS retry_log_time_index ON retry_log 1825 | (repo, time DESC) 1826 | ''') 1827 | 1828 | # manual DB migration :/ 1829 | try: 1830 | db_query(db, 'SELECT treeclosed_src FROM repos LIMIT 0') 1831 | except sqlite3.OperationalError: 1832 | db_query(db, 'ALTER TABLE repos ADD COLUMN treeclosed_src TEXT') 1833 | try: 1834 | db_query(db, 'SELECT squash FROM pull LIMIT 0') 1835 | except sqlite3.OperationalError: 1836 | db_query(db, 'ALTER TABLE pull ADD COLUMN squash INT') 1837 | 1838 | for repo_label, repo_cfg in cfg['repo'].items(): 1839 | repo_cfgs[repo_label] = repo_cfg 1840 | repo_labels[repo_cfg['owner'], repo_cfg['name']] = repo_label 1841 | 1842 | # If test-on-fork is enabled point both the main repo and the fork to 1843 | # the same homu "repository". This will allow events coming from both 1844 | # GitHub repositories to be processed the same way. 1845 | if 'test-on-fork' in repo_cfg: 1846 | tof = repo_cfg['test-on-fork'] 1847 | repo_labels[tof['owner'], tof['name']] = repo_label 1848 | 1849 | repo_states = {} 1850 | repos[repo_label] = Repository(None, repo_label, db) 1851 | 1852 | db_query( 1853 | db, 1854 | 'SELECT num, head_sha, status, title, body, head_ref, base_ref, assignee, approved_by, priority, try_, rollup, squash, delegate, merge_sha FROM pull WHERE repo = ?', # noqa 1855 | [repo_label]) 1856 | for num, head_sha, status, title, body, head_ref, base_ref, assignee, approved_by, priority, try_, rollup, squash, delegate, merge_sha in db.fetchall(): # noqa 1857 | state = PullReqState(num, head_sha, status, db, repo_label, mergeable_que, gh, repo_cfg['owner'], repo_cfg['name'], repo_cfg.get('labels', {}), repos, repo_cfg.get('test-on-fork')) # noqa 1858 | state.title = title 1859 | state.body = body 1860 | state.head_ref = head_ref 1861 | state.base_ref = base_ref 1862 | state.assignee = assignee 1863 | 1864 | state.approved_by = approved_by 1865 | state.priority = int(priority) 1866 | state.try_ = bool(try_) 1867 | state.rollup = rollup 1868 | state.squash = bool(squash) 1869 | state.delegate = delegate 1870 | builders = [] 1871 | if merge_sha: 1872 | if 'buildbot' in repo_cfg: 1873 | builders += repo_cfg['buildbot']['builders'] 1874 | if 'travis' in repo_cfg: 1875 | builders += ['travis'] 1876 | if 'status' in repo_cfg: 1877 | builders += ['status-' + key for key, value in repo_cfg['status'].items() if 'context' in value] # noqa 1878 | if 'checks' in repo_cfg: 1879 | builders += ['checks-' + key for key, value in repo_cfg['checks'].items() if 'name' in value] # noqa 1880 | if len(builders) == 0: 1881 | raise RuntimeError('Invalid configuration') 1882 | 1883 | state.init_build_res(builders, use_db=False) 1884 | state.merge_sha = merge_sha 1885 | 1886 | elif state.status == 'pending': 1887 | # FIXME: There might be a better solution 1888 | state.status = '' 1889 | 1890 | state.save() 1891 | 1892 | repo_states[num] = state 1893 | 1894 | states[repo_label] = repo_states 1895 | 1896 | db_query( 1897 | db, 1898 | 'SELECT repo, num, builder, res, url, merge_sha FROM build_res') 1899 | for repo_label, num, builder, res, url, merge_sha in db.fetchall(): 1900 | try: 1901 | state = states[repo_label][num] 1902 | if builder not in state.build_res: 1903 | raise KeyError 1904 | if state.merge_sha != merge_sha: 1905 | raise KeyError 1906 | except KeyError: 1907 | db_query( 1908 | db, 1909 | 'DELETE FROM build_res WHERE repo = ? AND num = ? AND builder = ?', # noqa 1910 | [repo_label, num, builder]) 1911 | continue 1912 | 1913 | state.build_res[builder] = { 1914 | 'res': bool(res) if res is not None else None, 1915 | 'url': url, 1916 | } 1917 | 1918 | db_query(db, 'SELECT repo, num, mergeable FROM mergeable') 1919 | for repo_label, num, mergeable in db.fetchall(): 1920 | try: 1921 | state = states[repo_label][num] 1922 | except KeyError: 1923 | db_query( 1924 | db, 1925 | 'DELETE FROM mergeable WHERE repo = ? AND num = ?', 1926 | [repo_label, num]) 1927 | continue 1928 | 1929 | state.mergeable = bool(mergeable) if mergeable is not None else None 1930 | 1931 | db_query(db, 'SELECT repo FROM pull GROUP BY repo') 1932 | for repo_label, in db.fetchall(): 1933 | if repo_label not in repos: 1934 | db_query(db, 'DELETE FROM pull WHERE repo = ?', [repo_label]) 1935 | 1936 | queue_handler_lock = Lock() 1937 | 1938 | def queue_handler(): 1939 | with queue_handler_lock: 1940 | return process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, git_cfg) # noqa 1941 | 1942 | os.environ['GIT_SSH'] = os.path.join(os.path.dirname(__file__), 'git_helper.py') # noqa 1943 | os.environ['GIT_EDITOR'] = 'cat' 1944 | 1945 | from . import server 1946 | Thread( 1947 | target=server.start, 1948 | args=[ 1949 | cfg, 1950 | states, 1951 | queue_handler, 1952 | repo_cfgs, 1953 | repos, 1954 | logger, 1955 | buildbot_slots, 1956 | my_username, 1957 | db, 1958 | repo_labels, 1959 | mergeable_que, 1960 | gh, 1961 | ]).start() 1962 | 1963 | Thread(target=fetch_mergeability, args=[mergeable_que]).start() 1964 | 1965 | queue_handler() 1966 | 1967 | 1968 | if __name__ == '__main__': 1969 | main() 1970 | --------------------------------------------------------------------------------