├── src └── hubcast │ ├── __init__.py │ ├── web │ ├── __init__.py │ ├── github │ │ ├── __init__.py │ │ ├── utils.py │ │ ├── handler.py │ │ └── routes.py │ ├── gitlab │ │ ├── __init__.py │ │ ├── handler.py │ │ └── routes.py │ └── comments.py │ ├── repos │ ├── __init__.py │ └── config.py │ ├── account_map │ ├── __init__.py │ ├── abc.py │ └── file.py │ ├── clients │ ├── github │ │ ├── __init__.py │ │ ├── auth.py │ │ └── client.py │ ├── gitlab │ │ ├── __init__.py │ │ ├── auth.py │ │ └── client.py │ └── utils.py │ ├── config.py │ ├── __main__.py │ └── logging.py ├── .github ├── workflows │ ├── requirements │ │ ├── security.txt │ │ ├── style.txt │ │ └── unit-tests.txt │ ├── label.yml │ ├── style.yml │ ├── unit-tests.yml │ ├── container.yml │ ├── security.yml │ └── ci.yml ├── ISSUE_TEMPLATE │ └── issue-feature-request.md ├── dependabot.yml └── labeler.yml ├── logo ├── logo.png └── logo.svg ├── spack └── repos │ └── spack_repo │ └── hubcast │ ├── repo.yaml │ └── packages │ └── py_hubcast │ └── package.py ├── .gitignore ├── spack.yaml ├── .envrc ├── Dockerfile ├── COPYRIGHT ├── logging_config.json ├── pyproject.toml ├── NOTICE ├── README.md ├── docs ├── spack-develop.md ├── CONTRIBUTING.md └── getting-started.md └── LICENSE /src/hubcast/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/hubcast/web/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/hubcast/repos/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/hubcast/account_map/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/workflows/requirements/security.txt: -------------------------------------------------------------------------------- 1 | bandit[sarif,toml]==1.8.6 2 | -------------------------------------------------------------------------------- /.github/workflows/requirements/style.txt: -------------------------------------------------------------------------------- 1 | ruff==0.13.2 2 | typos==1.40.0 3 | -------------------------------------------------------------------------------- /logo/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/hubcast/HEAD/logo/logo.png -------------------------------------------------------------------------------- /spack/repos/spack_repo/hubcast/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: 'hubcast' 3 | api: v2.0 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .env 3 | spack.lock 4 | .spack-env 5 | *.pem 6 | build-* 7 | .test-users.yml 8 | -------------------------------------------------------------------------------- /src/hubcast/web/github/__init__.py: -------------------------------------------------------------------------------- 1 | from .handler import GitHubHandler 2 | 3 | __all__ = ["GitHubHandler"] 4 | -------------------------------------------------------------------------------- /src/hubcast/web/gitlab/__init__.py: -------------------------------------------------------------------------------- 1 | from .handler import GitLabHandler 2 | 3 | __all__ = ["GitLabHandler"] 4 | -------------------------------------------------------------------------------- /src/hubcast/clients/github/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import GitHubClient, GitHubClientFactory 2 | 3 | __all__ = ["GitHubClient", "GitHubClientFactory"] 4 | -------------------------------------------------------------------------------- /src/hubcast/clients/gitlab/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import GitLabClient, GitLabClientFactory 2 | 3 | __all__ = ["GitLabClient", "GitLabClientFactory"] 4 | -------------------------------------------------------------------------------- /.github/workflows/requirements/unit-tests.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.13.2 2 | build==1.3.0 3 | cachetools==6.2.2 4 | coverage==7.12.0 5 | gidgethub==5.4.0 6 | gidgetlab==2.1.1 7 | hatchling==1.27.0 8 | pytest-asyncio==1.3.0 9 | pytest-mock==3.15.1 10 | pytest==8.4.1 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue-feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue/Feature Request 3 | about: Standard issue/feature request template 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Problem/Opportunity Statement 11 | 12 | 13 | ## What would success / a fix look like? 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | 8 | - package-ecosystem: "pip" 9 | directory: "/.github/workflows/requirements" 10 | schedule: 11 | interval: "monthly" 12 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | ci: 2 | - changed-files: 3 | - any-glob-to-any-file: 4 | - .github/** 5 | 6 | dependencies: 7 | - changed-files: 8 | - any-glob-to-any-file: 9 | - .github/workflows/requirements/** 10 | - pyproject.toml 11 | - spack/** 12 | 13 | docs: 14 | - changed-files: 15 | - any-glob-to-any-file: 16 | - docs/** 17 | - README.md 18 | -------------------------------------------------------------------------------- /src/hubcast/account_map/abc.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Union 3 | 4 | 5 | class AccountMap(ABC): 6 | """ 7 | An abstract interface defining an account map. 8 | """ 9 | 10 | @abstractmethod 11 | def __call__(self, github_user: str) -> Union[str, None]: 12 | """ 13 | Return the coorisponding gitlab_user for a given github_user if 14 | one exists. 15 | """ 16 | pass 17 | -------------------------------------------------------------------------------- /spack.yaml: -------------------------------------------------------------------------------- 1 | # This is a Spack Environment file. 2 | # 3 | # It describes a set of packages to be installed, along with 4 | # configuration settings. 5 | spack: 6 | repos: 7 | hubcast: spack/repos/spack_repo/hubcast 8 | specs: 9 | - py-pip 10 | - py-hubcast 11 | - py-bandit 12 | - py-ruff 13 | - smee-client 14 | - typos 15 | view: true 16 | concretizer: 17 | unify: true 18 | develop: 19 | py-hubcast: 20 | spec: py-hubcast@=main 21 | path: . 22 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------ 2 | # Load Development Spack Environment (If Spack is installed.) 3 | # 4 | # Run 'direnv allow' from within the cloned repository to automatically 5 | # load the spack environment when you enter the directory. 6 | #------------------------------------------------------------------------ 7 | if type spack &>/dev/null; then 8 | . $SPACK_ROOT/share/spack/setup-env.sh 9 | spack env activate -d . 10 | fi 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-alpine 2 | 3 | ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 4 | WORKDIR /app 5 | 6 | # copy build deps 7 | COPY pyproject.toml README.md ./ 8 | COPY src ./src 9 | # hubcast default logging config 10 | COPY logging_config.json ./ 11 | 12 | # create venv and build hubcast and deps 13 | RUN python -m venv /venv \ 14 | && /venv/bin/pip install --upgrade pip setuptools wheel \ 15 | && /venv/bin/pip install --no-cache-dir /app \ 16 | && rm -rf /root/.cache /tmp/* 17 | 18 | ENV PATH="/venv/bin:$PATH" 19 | ENTRYPOINT ["python", "-m", "hubcast"] 20 | -------------------------------------------------------------------------------- /src/hubcast/repos/config.py: -------------------------------------------------------------------------------- 1 | class RepoConfig: 2 | def __init__( 3 | self, 4 | fullname: str, 5 | dest_org: str, 6 | dest_name: str, 7 | check_name: str = "gitlab-ci", 8 | check_type: str = "pipeline", 9 | create_mr: bool = False, 10 | delete_closed: bool = True, 11 | ): 12 | self.fullname = fullname 13 | self.dest_org = dest_org 14 | self.dest_name = dest_name 15 | self.check_name = check_name 16 | self.check_type = check_type 17 | self.create_mr = create_mr 18 | self.delete_closed = delete_closed 19 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Intellectual Property Notice 2 | ------------------------------ 3 | 4 | Hubcast is licensed under the Apache License, Version 2.0 (LICENSE-APACHE 5 | or http://www.apache.org/licenses/LICENSE-2.0). 6 | 7 | Copyrights and patents in the Benchpark project are retained by contributors. 8 | No copyright assignment is required to contribute to Benchpark. 9 | 10 | 11 | SPDX usage 12 | ------------ 13 | 14 | Individual files contain SPDX tags instead of the full license text. 15 | This enables machine processing of license information based on the SPDX 16 | License Identifiers that are available here: https://spdx.org/licenses/ 17 | -------------------------------------------------------------------------------- /src/hubcast/web/comments.py: -------------------------------------------------------------------------------- 1 | def help_message(bot_user: str): 2 | return f""" 3 | You can interact with me in many ways! 4 | 5 | - `@{bot_user} help`: see this message 6 | - `@{bot_user} approve`: sync this pull request with the destination repository and trigger a new pipeline 7 | - `@{bot_user} run pipeline`: request a new run of the GitLab CI pipeline for any reason 8 | - `@{bot_user} restart failed jobs`: restart any failed jobs in the latest CI pipeline 9 | 10 | If you are an outside contributor to this repository, a maintainer will need to approve and run pipelines on your behalf. 11 | 12 | For assistance and bug reports, open an issue [here](https://github.com/llnl/hubcast/issues). 13 | """ 14 | -------------------------------------------------------------------------------- /.github/workflows/label.yml: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------- 2 | # DO NOT modify unless you really know what you are doing. 3 | # 4 | # See https://stackoverflow.com/a/74959635 for more info. 5 | # Talk to @alecbcs if you have questions/are not sure of a change's 6 | # possible impact to security. 7 | #----------------------------------------------------------------------- 8 | name: label 9 | on: 10 | pull_request_target: 11 | branches: 12 | - main 13 | 14 | jobs: 15 | pr: 16 | runs-on: ubuntu-latest 17 | permissions: 18 | contents: read 19 | pull-requests: write 20 | steps: 21 | - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b 22 | -------------------------------------------------------------------------------- /.github/workflows/style.yml: -------------------------------------------------------------------------------- 1 | name: Linting & Style Checks 2 | on: 3 | workflow_call: 4 | 5 | permissions: 6 | contents: read 7 | 8 | jobs: 9 | lint: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 13 | 14 | - name: Set up Python 3.10 15 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 16 | with: 17 | python-version: '3.10' 18 | cache: 'pip' 19 | cache-dependency-path: '.github/workflows/requirements/style.txt' 20 | 21 | - name: Install Python dependencies 22 | run: | 23 | pip install -r .github/workflows/requirements/style.txt 24 | 25 | - name: Run Ruff 26 | run: | 27 | ruff check --diff 28 | ruff check --select I --diff 29 | ruff format --check --diff 30 | 31 | - name: Run Typos 32 | run: typos 33 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | on: 3 | workflow_call: 4 | 5 | permissions: 6 | contents: read 7 | 8 | jobs: 9 | ubuntu: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | python-version: ['3.10', '3.13'] 14 | steps: 15 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 16 | - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | cache: 'pip' 20 | cache-dependency-path: | 21 | 'pyproject.toml' 22 | '.github/workflows/requirements/unit-tests.txt' 23 | 24 | - name: Install Python dependencies 25 | run: | 26 | pip install . 27 | pip install -r .github/workflows/requirements/unit-tests.txt 28 | 29 | - name: Run Unit Tests with Pytest 30 | run: | 31 | python3 -m build # build package for compatibility with publish.yml 32 | python3 -m pytest 33 | -------------------------------------------------------------------------------- /src/hubcast/web/github/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Dict 3 | 4 | from hubcast.clients.github import GitHubClient 5 | from hubcast.clients.github.client import InvalidConfigYAMLError 6 | from hubcast.repos.config import RepoConfig 7 | 8 | config_cache = dict() 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | def create_config(fullname: str, data: Dict) -> RepoConfig: 13 | return RepoConfig( 14 | fullname=fullname, 15 | dest_org=data["Repo"]["owner"], 16 | dest_name=data["Repo"]["name"], 17 | ) 18 | 19 | 20 | async def get_repo_config(gh: GitHubClient, fullname: str, refresh: bool = False): 21 | if fullname in config_cache and not refresh: 22 | config = config_cache[fullname] 23 | else: 24 | try: 25 | data = await gh.get_repo_config() 26 | except InvalidConfigYAMLError: 27 | log.exception("Repo config parse failed") 28 | 29 | config = create_config(fullname, data) 30 | config_cache[fullname] = config 31 | 32 | return config 33 | -------------------------------------------------------------------------------- /.github/workflows/container.yml: -------------------------------------------------------------------------------- 1 | name: Container Build & Publish 2 | on: 3 | workflow_call: 4 | 5 | permissions: 6 | contents: read 7 | packages: write 8 | 9 | jobs: 10 | build-release: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 14 | - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 15 | - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 16 | 17 | - name: Login to GitHub Container Registry 18 | uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef 19 | with: 20 | registry: ghcr.io 21 | username: ${{ github.actor }} 22 | password: ${{ secrets.GITHUB_TOKEN }} 23 | 24 | - name: Build and push image 25 | uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 26 | with: 27 | context: . 28 | platforms: linux/amd64,linux/arm64 29 | push: ${{ github.ref == 'refs/heads/main' }} 30 | tags: ghcr.io/${{ github.repository }} 31 | -------------------------------------------------------------------------------- /spack/repos/spack_repo/hubcast/packages/py_hubcast/package.py: -------------------------------------------------------------------------------- 1 | # Copyright Spack Project Developers. See COPYRIGHT file for details. 2 | # 3 | # SPDX-License-Identifier: (Apache-2.0 OR MIT) 4 | 5 | from spack_repo.builtin.build_systems.python import PythonPackage 6 | 7 | from spack.package import * 8 | 9 | 10 | class PyHubcast(PythonPackage): 11 | """An event driven synchronization application for bridging GitHub and GitLab.""" 12 | 13 | homepage = "https://github.com/LLNL/hubcast" 14 | git = "https://github.com/LLNL/hubcast.git" 15 | 16 | maintainers("alecbcs", "cmelone") 17 | 18 | license("Apache-2.0") 19 | 20 | version("main", branch="main") 21 | 22 | depends_on("python@3.10:", type=("build", "run")) 23 | 24 | depends_on("py-hatchling", type="build") 25 | 26 | depends_on("py-aiohttp", type=("build", "run")) 27 | depends_on("py-aiojobs", type=("build", "run")) 28 | depends_on("py-gidgethub", type=("build", "run")) 29 | depends_on("py-gidgetlab+aiohttp", type=("build", "run")) 30 | depends_on("py-repligit", type=("build", "run")) 31 | depends_on("py-pyyaml", type=("build", "run")) 32 | -------------------------------------------------------------------------------- /logging_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "disable_existing_loggers": false, 4 | "formatters": { 5 | "default": { 6 | "()": "hubcast.logging.HubcastConsoleFormatter", 7 | "format": "[%(asctime)s] %(levelname)s %(name)s: %(message)s", 8 | "datefmt": "%Y-%m-%dT%H:%M:%S%z" 9 | }, 10 | "json": { 11 | "()": "hubcast.logging.HubcastJSONFormatter", 12 | "fmt_keys": { 13 | "level": "levelname", 14 | "message": "message", 15 | "timestamp": "timestamp", 16 | "logger": "name" 17 | } 18 | } 19 | }, 20 | "handlers": { 21 | "console": { 22 | "class": "logging.StreamHandler", 23 | "formatter": "default", 24 | "stream": "ext://sys.stdout" 25 | }, 26 | "json_stdout": { 27 | "class": "logging.StreamHandler", 28 | "formatter": "json", 29 | "stream": "ext://sys.stdout" 30 | } 31 | }, 32 | "root": { 33 | "level": "DEBUG", 34 | "handlers": [ 35 | "json_stdout" 36 | ] 37 | } 38 | } -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling >= 1.26"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "hubcast" 7 | version = "0.0.1" 8 | dependencies = [ 9 | "aiohttp", 10 | "aiojobs", 11 | "cachetools", 12 | "gidgethub", 13 | "gidgetlab", 14 | "repligit[aiohttp]", 15 | "pyyaml", 16 | ] 17 | authors = [ 18 | { name = "Alec Scott" }, 19 | ] 20 | description = "An event driven synchronization application for bridging GitHub and GitLab." 21 | readme = "README.md" 22 | requires-python = ">=3.10" 23 | classifiers = [ 24 | "Programming Language :: Python :: 3", 25 | "Operating System :: OS Independent", 26 | ] 27 | license = "Apache-2.0" 28 | keywords = ["github", "gitlab", "mirroring", "ci"] 29 | 30 | [project.urls] 31 | Homepage = "https://github.com/llnl/hubcast" 32 | Issues = "https://github.com/llnl/hubcast/issues" 33 | 34 | [project.optional-dependencies] 35 | dev = [ 36 | "pytest", 37 | "pytest-asyncio", 38 | "pytest-mock" 39 | ] 40 | 41 | [tool.pytest.ini_options] 42 | pythonpath = [ 43 | "src" 44 | ] 45 | 46 | [tool.ruff] 47 | line-length = 88 48 | exclude = ["spack"] 49 | 50 | [tool.bandit] 51 | exclude_dirs = [] 52 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. 22 | -------------------------------------------------------------------------------- /src/hubcast/clients/utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Awaitable, Callable, Tuple 3 | 4 | 5 | class TokenCache: 6 | """ 7 | Cache for web tokens with an expiration. 8 | """ 9 | 10 | def __init__(self) -> None: 11 | self._tokens = {} 12 | 13 | async def get( 14 | self, 15 | name: str, 16 | renew: Callable[[], Awaitable[Tuple[float, str]]], 17 | time_needed: int = 60, 18 | ) -> str: 19 | """ 20 | Get a cached token, or renew as needed. 21 | 22 | Parameters 23 | --------- 24 | name: str 25 | An identifying name of a token to get from the cache. 26 | renew: Callable[[], Awaitable[Tuple[float, str]]] 27 | A function to call in order to generate a new token if the cache 28 | is stale. 29 | time_needed: int 30 | The number of seconds a token will be needed. Thus any token that 31 | expires during this window should be disregarded and renewed. 32 | """ 33 | expires, token = self._tokens.get(name, (0, "")) 34 | 35 | now = time.time() 36 | if expires < now + time_needed: 37 | expires, token = await renew() 38 | self._tokens[name] = (expires, token) 39 | 40 | return token 41 | -------------------------------------------------------------------------------- /src/hubcast/account_map/file.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Union 2 | 3 | import yaml 4 | 5 | from .abc import AccountMap 6 | 7 | 8 | class FileMapError(Exception): 9 | pass 10 | 11 | 12 | class FileMap(AccountMap): 13 | """ 14 | A simple user map importing from a YAML file of the form. 15 | 16 | Users: 17 | github_user: gitlab_user 18 | github_user2: gitlab_user2 19 | 20 | Attributes 21 | ---------- 22 | path: str 23 | A filepath to the users.yml defining a usermapping. 24 | """ 25 | 26 | path: str 27 | users: Dict[str, str] 28 | 29 | def __init__(self, path: str): 30 | """ 31 | Constructor, path to read from and generate a simple account 32 | mapping between services. 33 | """ 34 | self.path = path 35 | 36 | try: 37 | with open(path, "r") as f: 38 | data = yaml.safe_load(f) 39 | self.users = data["Users"] 40 | except FileNotFoundError: 41 | raise FileMapError(f"File map not found. path={path}") 42 | except yaml.YAMLError: 43 | raise FileMapError(f"Failed to parse file map. path={path}") 44 | 45 | def __call__(self, github_user: str) -> Union[str, None]: 46 | """ 47 | Return the gitlab_user for a github_user if one exists. 48 | """ 49 | return self.users.get(github_user) 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 | Hubcast logo 4 |
5 |

6 | 7 | **[Features]   •   [Getting Started]   •   [Config]   •   [Contributing]   •   [Changelog]** 8 | 9 | [Features]: #features 10 | [Getting Started]: /docs/getting-started.md 11 | [Config]: /docs/getting-started.md 12 | [Contributing]: /docs/CONTRIBUTING.md 13 | [Changelog]: https://github.com/LLNL/hubcast/releases 14 | 15 |
16 | 17 | Hubcast is an event driven synchronization application for bridging GitHub and GitLab. It automates various workflow tasks and handles jobs like: 18 | 19 | - Syncing branches from GitHub to GitLab. 20 | - Reporting CI job statuses back to GitHub from GitLab Workflow Runs. 21 | 22 | 23 | ## License 24 | 25 | Licensed under the Apache License, Version 2.0 (the "License"); 26 | you may not use this file except in compliance with the License. 27 | You may obtain a copy of the License at 28 | 29 | http://www.apache.org/licenses/LICENSE-2.0 30 | 31 | Unless required by applicable law or agreed to in writing, software 32 | distributed under the License is distributed on an "AS IS" BASIS, 33 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 34 | See the License for the specific language governing permissions and 35 | limitations under the License. 36 | 37 | SPDX-License-Identifier: Apache-2.0 38 | 39 | LLNL-CODE-847946 40 | -------------------------------------------------------------------------------- /docs/spack-develop.md: -------------------------------------------------------------------------------- 1 | # Developing Hubcast with Spack (Recommended) 2 | ## Prerequisites 3 | You'll need to install Spack if you haven't already. You can clone Spack from 4 | GitHub by running the following, 5 | 6 | ```bash 7 | $ git clone -c feature.manyFiles=true https://github.com/spack/spack.git 8 | $ cd spack/ 9 | ``` 10 | 11 | Next we'll need to load Spack into our shell. (Add one of the following lines 12 | -- prepended by the directory you cloned spack into -- to your 13 | `.zshrc`, `.bashrc`, or equivalent to make it permanent.) 14 | 15 | ```bash 16 | # For bash/zsh/sh 17 | $ . spack/share/spack/setup-env.sh 18 | 19 | # For tcsh/csh 20 | $ source spack/share/spack/setup-env.csh 21 | 22 | # For fish 23 | $ . spack/share/spack/setup-env.fish 24 | ``` 25 | 26 | ## Activating the Development Environment 27 | Activate the Spack environment by entering the following, 28 | ```bash 29 | $ cd path/to/hubcast 30 | $ spack env activate -d . 31 | ``` 32 | 33 | > [!TIP] 34 | > If you've got [direnv](https://direnv.net) installed on your system 35 | > you can run `direnv allow` to automatically load the spack environment 36 | > when you cd into the repository in the future. 37 | 38 | ## Installing the Development Environment 39 | Install Hubcast's development dependencies with Spack by running 40 | the following, 41 | ```bash 42 | $ spack install 43 | ``` 44 | 45 | ## Upgrading the Development Environment 46 | To update your Spack environment run, 47 | ```bash 48 | $ spack concretize --force --fresh 49 | $ spack install 50 | ``` 51 | -------------------------------------------------------------------------------- /src/hubcast/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Optional 3 | 4 | 5 | class ConfigError(Exception): 6 | pass 7 | 8 | 9 | class Config: 10 | def __init__(self): 11 | self.port = int(env_get("HC_PORT", default="8080")) 12 | 13 | self.account_map_type = env_get("HC_ACCOUNT_MAP_TYPE") 14 | self.account_map_path = env_get("HC_ACCOUNT_MAP_PATH") 15 | self.logging_config_path = env_get("HC_LOGGING_CONFIG_PATH") 16 | 17 | self.gh = GitHubConfig() 18 | self.gl = GitLabConfig() 19 | 20 | 21 | class GitHubConfig: 22 | def __init__(self): 23 | self.app_id = env_get("HC_GH_APP_IDENTIFIER") 24 | self.privkey = env_get("HC_GH_PRIVATE_KEY") 25 | self.requester = env_get("HC_GH_REQUESTER") 26 | self.webhook_secret = env_get("HC_GH_SECRET") 27 | self.bot_user = env_get("HC_GH_BOT_USER") 28 | 29 | 30 | class GitLabConfig: 31 | def __init__(self): 32 | self.instance_url = env_get("HC_GL_URL") 33 | # requester identifies the app making requests, it doesn't 34 | # perform any auth function but is included in user-agent 35 | self.requester = env_get("HC_GL_REQUESTER") 36 | self.token = env_get("HC_GL_TOKEN") 37 | self.token_type = env_get("HC_GL_TOKEN_TYPE", default="impersonation") 38 | self.webhook_secret = env_get("HC_GL_SECRET") 39 | self.callback_url = env_get("HC_GL_CALLBACK_URL") 40 | 41 | 42 | def env_get(key: str, default: Optional[str] = None) -> str: 43 | value = os.environ.get(key) or default 44 | if not value: 45 | raise ConfigError(f"Required environment variable not found: {key}") 46 | 47 | return value 48 | -------------------------------------------------------------------------------- /src/hubcast/web/github/handler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from aiohttp import web 4 | from aiojobs.aiohttp import spawn 5 | from gidgethub import sansio 6 | 7 | from .routes import router 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | class GitHubHandler: 13 | def __init__( 14 | self, webhook_secret, account_map, github_client_factory, gitlab_client_factory 15 | ): 16 | self.webhook_secret = webhook_secret 17 | self.account_map = account_map 18 | self.gh = github_client_factory 19 | self.gl = gitlab_client_factory 20 | 21 | async def handle(self, request): 22 | try: 23 | # read the GitHub webhook payload 24 | body = await request.read() 25 | event = sansio.Event.from_http( 26 | request.headers, body, secret=self.webhook_secret 27 | ) 28 | 29 | log.info( 30 | "GitHub webhook received", 31 | extra={"event_type": event.event, "delivery_id": event.delivery_id}, 32 | ) 33 | 34 | github_user = event.data["sender"]["login"] 35 | gitlab_user = self.account_map(github_user) 36 | 37 | if gitlab_user is None: 38 | log.info("Unauthorized GitHub user", extra={"github_user": github_user}) 39 | return web.Response(status=200) 40 | 41 | gh_repo_owner = event.data["repository"]["owner"]["login"] 42 | gh_repo = event.data["repository"]["name"] 43 | 44 | gh = self.gh.create_client(gh_repo_owner, gh_repo) 45 | gl = self.gl.create_client(gitlab_user) 46 | 47 | await spawn(request, router.dispatch(event, gh, gl, gitlab_user)) 48 | 49 | # return a "Success" 50 | return web.Response(status=200) 51 | except Exception: 52 | log.exception("Failed to handle Github webhook") 53 | return web.Response(status=500) 54 | -------------------------------------------------------------------------------- /src/hubcast/web/gitlab/handler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from aiohttp import web 4 | from aiojobs.aiohttp import spawn 5 | from gidgetlab import sansio 6 | from gidgetlab.exceptions import ValidationFailure 7 | 8 | from hubcast.clients.github import GitHubClientFactory 9 | 10 | from .routes import router 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class GitLabHandler: 16 | def __init__(self, webhook_secret: str, github_client_factory: GitHubClientFactory): 17 | self.webhook_secret = webhook_secret 18 | self.github_client_factory = github_client_factory 19 | 20 | async def handle(self, request): 21 | try: 22 | # read the GitLab webhook payload 23 | body = await request.read() 24 | event = sansio.Event.from_http( 25 | request.headers, body, secret=self.webhook_secret 26 | ) 27 | log.info("GitLab webhook received", extra={"event_type": event.event}) 28 | 29 | # get coorisponding GitHub repo owner and name from event 30 | # request variables 31 | gh_repo_owner = request.rel_url.query["gh_owner"] 32 | gh_repo = request.rel_url.query["gh_repo"] 33 | 34 | github_client = self.github_client_factory.create_client( 35 | gh_repo_owner, gh_repo 36 | ) 37 | 38 | gh_check_name = request.rel_url.query["gh_check"] 39 | 40 | await spawn( 41 | request, 42 | router.dispatch(event, github_client, gh_check_name), 43 | ) 44 | 45 | # return a "Success" 46 | return web.Response(status=200) 47 | except ValidationFailure: 48 | log.exception( 49 | "Failed to validate Gitlab webhook request", 50 | ) 51 | return web.Response(status=500) 52 | 53 | except Exception: 54 | log.exception("Failed to handle GitLab webhook") 55 | return web.Response(status=500) 56 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: Security Checks 2 | on: 3 | workflow_call: 4 | 5 | permissions: 6 | contents: read 7 | security-events: write 8 | 9 | jobs: 10 | bandit-scan: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 14 | 15 | # https://bandit.readthedocs.io/en/latest/faq.html#under-which-version-of-python-should-i-install-bandit 16 | - name: Set up Python 3.10 17 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 18 | with: 19 | python-version: '3.10' 20 | cache: 'pip' 21 | cache-dependency-path: '.github/workflows/requirements/security.txt' 22 | 23 | - name: Install Python dependencies 24 | run: pip install -r .github/workflows/requirements/security.txt 25 | 26 | - name: Run Bandit 27 | run: bandit -c pyproject.toml -r src -f sarif -o results.sarif 28 | 29 | # upload security results to github; a bot will add inline comments to 30 | # the PR if any issues are found 31 | - name: Upload SARIF file 32 | if: always() 33 | uses: github/codeql-action/upload-sarif@fe4161a26a8629af62121b670040955b330f9af2 34 | with: 35 | sarif_file: results.sarif 36 | category: bandit 37 | 38 | codeql-analyze: 39 | runs-on: ubuntu-latest 40 | strategy: 41 | fail-fast: false 42 | matrix: 43 | language: [actions, python] 44 | steps: 45 | - name: Checkout repository 46 | uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 47 | 48 | - name: Initialize CodeQL 49 | uses: github/codeql-action/init@fe4161a26a8629af62121b670040955b330f9af2 50 | with: 51 | languages: ${{ matrix.language }} 52 | build-mode: none 53 | 54 | - name: Perform CodeQL Analysis 55 | uses: github/codeql-action/analyze@fe4161a26a8629af62121b670040955b330f9af2 56 | with: 57 | category: "/language:${{matrix.language}}" 58 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | ## Introduction 3 | Hubcast welcomes contributions via [Pull Requests](https://github.com/LLNL/hubcast/pulls). 4 | We've labeled beginner friendly (good first issue) tasks in the issue tracker, feel free 5 | to reach out and ask for help when getting started. 6 | 7 | For small changes (e.g. bug fixes), feel free to submit a PR. 8 | 9 | For larger architectural changes and new features, consider opening an 10 | [issue](https://github.com/LLNL/hubcast/issues/new?template=issue-feature-request.md) outlining your 11 | proposed contribution. 12 | 13 | ## Prerequisites 14 | Hubcast is written in python. You'll need a version of python and pip to 15 | install the required dependencies and nodejs to install the 16 | [smee-client](https://www.npmjs.com/package/smee-client) to test the application locally. 17 | 18 | You can install the full development environment using [Spack](spack-develop.md). 19 | 20 | ## Development 21 | After cloning the repository you'll need to follow the [Getting Started](getting-started.md) 22 | documentation to setup a testing, 23 | 1. GitHub Repo 24 | 2. GitHub App 25 | 3. GitLab Repo 26 | 4. GitLab Repo Webhook 27 | 5. GitLab Repo Access Token 28 | 29 | > [!TIP] 30 | > If you're developing locally you can use [smee.io](https://smee.io) to relay 31 | > webhooks to your local machine. Just click "Start a new channel" & then run 32 | > the following substituting your channel url as the argument and GitHub App 33 | > endpoint. 34 | > 35 | > ```bash 36 | > $ smee -u https://smee.io/reDaCTed 37 | > ``` 38 | 39 | ## Project Structure 40 | ```bash 41 | . 42 | ├── LICENSE 43 | ├── README.md 44 | ├── docs # ---------> project documentation 45 | ├── hubcast # ------> python application 46 | ├── pyproject.toml 47 | ├── spack.lock 48 | └── spack.yaml -----> spack development environment 49 | ``` 50 | ```bash 51 | hubcast 52 | ├── __main__.py # --> hubcast entrypoint and config setup 53 | ├── auth # ---------> authentication library for GitHub/GitLab 54 | ├── github.py # ----> GitHub router setup 55 | ├── gitlab.py # ----> GitLab router setup 56 | ├── routes # -------> GitHub & GitLab event routing logic 57 | └── utils # --------> Git and common application utilities 58 | ``` 59 | -------------------------------------------------------------------------------- /src/hubcast/web/gitlab/routes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Any 3 | 4 | from gidgetlab import routing, sansio 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class GitLabRouter(routing.Router): 10 | """ 11 | Custom router to handle common interactions for hubcast 12 | """ 13 | 14 | async def dispatch(self, event: sansio.Event, *args: Any, **kwargs: Any) -> None: 15 | """Dispatch an event to all registered function(s).""" 16 | 17 | found_callbacks = [] 18 | try: 19 | found_callbacks.extend(self._shallow_routes[event.event]) 20 | except KeyError: 21 | pass 22 | try: 23 | details = self._deep_routes[event.event] 24 | except KeyError: 25 | pass 26 | else: 27 | for data_key, data_values in details.items(): 28 | if data_key in event.object_attributes: 29 | event_value = event.object_attributes[data_key] 30 | if event_value in data_values: 31 | found_callbacks.extend(data_values[event_value]) 32 | for callback in found_callbacks: 33 | try: 34 | await callback(event, *args, **kwargs) 35 | except Exception: 36 | # this catches errors related to processing of webhook events 37 | log.exception( 38 | "Failed to process GitLab webhook event", 39 | extra={ 40 | "event_type": event.event, 41 | }, 42 | ) 43 | 44 | 45 | router = GitLabRouter() 46 | 47 | 48 | @router.register("Pipeline Hook", status="pending") 49 | @router.register("Pipeline Hook", status="running") 50 | @router.register("Pipeline Hook", status="success") 51 | @router.register("Pipeline Hook", status="failed") 52 | @router.register("Pipeline Hook", status="canceled") 53 | async def status_relay(event, gh, gh_check_name, *arg, **kwargs): 54 | """Relay status of a GitLab pipeline back to GitHub.""" 55 | # get ref from event 56 | ref = event.data["object_attributes"]["sha"] 57 | 58 | # get status from event 59 | ci_status = event.data["object_attributes"]["status"] 60 | pipeline_url = event.data["object_attributes"]["url"] 61 | 62 | # https://docs.github.com/en/rest/guides/using-the-rest-api-to-interact-with-checks#about-check-suites 63 | # https://docs.gitlab.com/api/pipelines/#list-project-pipelines -> status description 64 | 65 | # translate between GitLab and GitHub statuses 66 | if ci_status == "pending": 67 | status = "queued" 68 | elif ci_status == "running": 69 | status = "in_progress" 70 | elif ci_status == "failed": 71 | status = "failure" 72 | elif ci_status == "canceled": 73 | status = "cancelled" 74 | else: 75 | status = ci_status 76 | 77 | await gh.set_check_status(ref, gh_check_name, status, pipeline_url) 78 | -------------------------------------------------------------------------------- /src/hubcast/__main__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import logging.config 4 | import os 5 | import sys 6 | 7 | from aiohttp import web 8 | from aiojobs.aiohttp import setup 9 | 10 | from hubcast.account_map.file import FileMap, FileMapError 11 | from hubcast.clients.github import GitHubClientFactory 12 | from hubcast.clients.gitlab import GitLabClientFactory 13 | from hubcast.config import Config, ConfigError 14 | from hubcast.web.github import GitHubHandler 15 | from hubcast.web.gitlab import GitLabHandler 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | 20 | def main(): 21 | app = web.Application() 22 | 23 | try: 24 | conf = Config() 25 | except ConfigError as exc: 26 | log.error(exc) 27 | sys.exit(1) 28 | 29 | if os.path.exists(conf.logging_config_path): 30 | try: 31 | with open(conf.logging_config_path) as f: 32 | logging_config = json.load(f) 33 | logging.config.dictConfig(logging_config) 34 | except ( 35 | json.decoder.JSONDecodeError, 36 | # calls to logging.config.dictConfig will raise the following exceptions (cf stdlib docs): 37 | ValueError, 38 | TypeError, 39 | AttributeError, 40 | ImportError, 41 | ) as exc: 42 | log.error(exc) 43 | sys.exit(1) 44 | else: 45 | logging.basicConfig(level=logging.INFO) 46 | 47 | # error if we're unable to initialize an account map 48 | if conf.account_map_type == "file": 49 | try: 50 | account_map = FileMap(conf.account_map_path) 51 | except FileMapError: 52 | log.exception("Error initializing file account map") 53 | sys.exit(1) 54 | else: 55 | log.error( 56 | "Unknown account map type", 57 | extra={"account_map_type": conf.account_map_type}, 58 | ) 59 | sys.exit(1) 60 | 61 | gh_client_factory = GitHubClientFactory( 62 | conf.gh.app_id, conf.gh.privkey, conf.gh.requester, conf.gh.bot_user 63 | ) 64 | gl_client_factory = GitLabClientFactory( 65 | conf.gl.instance_url, 66 | conf.gl.requester, 67 | conf.gl.token, 68 | conf.gl.callback_url, 69 | conf.gl.webhook_secret, 70 | conf.gl.token_type, 71 | ) 72 | 73 | gh_handler = GitHubHandler( 74 | conf.gh.webhook_secret, 75 | account_map, 76 | gh_client_factory, 77 | gl_client_factory, 78 | ) 79 | 80 | gl_handler = GitLabHandler( 81 | conf.gl.webhook_secret, 82 | gh_client_factory, 83 | ) 84 | 85 | log.info("Starting HTTP server") 86 | 87 | app.router.add_post("/v1/events/src/github", gh_handler.handle) 88 | app.router.add_post("/v1/events/dest/gitlab", gl_handler.handle) 89 | 90 | setup(app) 91 | web.run_app( 92 | app, 93 | port=conf.port, 94 | access_log_format='"%r" %s %b "%{Referer}i" "%{User-Agent}i"', 95 | ) 96 | 97 | 98 | if __name__ == "__main__": 99 | main() 100 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | 10 | permissions: 11 | contents: read 12 | 13 | concurrency: 14 | group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | changes: 19 | runs-on: ubuntu-latest 20 | outputs: 21 | style: ${{ steps.filter.outputs.style }} 22 | security: ${{ steps.filter.outputs.security }} 23 | unit-tests: ${{ steps.filter.outputs.unit-tests }} 24 | container: ${{ steps.filter.outputs.container }} 25 | steps: 26 | - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # @v2 27 | if: ${{ github.event_name == 'push' }} 28 | with: 29 | fetch-depth: 0 30 | 31 | # For pull requests it's not necessary to checkout the code 32 | - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 33 | id: filter 34 | with: 35 | filters: | 36 | style: 37 | - '.github/**/*' 38 | - 'src/**/*' 39 | - 'pyproject.toml' 40 | security: 41 | - '.github/**/*' 42 | - 'src/**/*' 43 | - 'pyproject.toml' 44 | - '.bandit' 45 | container: 46 | - '.github/**/*' 47 | - 'src/**/*' 48 | - 'pyproject.toml' 49 | - 'Dockerfile' 50 | # unit-tests: 51 | # - '.github/**/*' 52 | # - 'src/**/*' 53 | # - 'tests/**/*' 54 | # - 'pyproject.toml' 55 | 56 | style: 57 | if: ${{ needs.changes.outputs.style == 'true' }} 58 | needs: changes 59 | uses: ./.github/workflows/style.yml 60 | 61 | security: 62 | if: ${{ needs.changes.outputs.security == 'true' }} 63 | needs: [changes, style] 64 | uses: ./.github/workflows/security.yml 65 | permissions: 66 | contents: read 67 | security-events: write 68 | 69 | # unit-tests: 70 | # if: ${{ needs.changes.outputs.unit-tests == 'true' }} 71 | # needs: [changes, style] 72 | # uses: ./.github/workflows/unit-tests.yml 73 | 74 | # coverage: 75 | # if: ${{ needs.changes.outputs.unit-tests == 'true' }} 76 | # needs: [changes, style, unit-tests] 77 | # uses: ./.github/workflows/coverage.yml 78 | 79 | container: 80 | if: ${{ needs.changes.outputs.container == 'true' }} 81 | needs: [changes, style, security] 82 | uses: ./.github/workflows/container.yml 83 | permissions: 84 | contents: read 85 | packages: write 86 | 87 | all: 88 | needs: 89 | - changes 90 | - style 91 | - security 92 | # - unit-tests 93 | # - coverage 94 | - container 95 | if: always() 96 | runs-on: ubuntu-latest 97 | steps: 98 | - name: Status summary 99 | run: | 100 | if ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}; then 101 | echo "One or more required jobs failed or were cancelled" 102 | exit 1 103 | else 104 | echo "All jobs completed successfully" 105 | fi 106 | -------------------------------------------------------------------------------- /src/hubcast/logging.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | import json 3 | import logging 4 | 5 | LOG_RECORD_BUILTIN_ATTRS = frozenset( 6 | { 7 | "args", 8 | "asctime", 9 | "created", 10 | "exc_info", 11 | "exc_text", 12 | "filename", 13 | "funcName", 14 | "levelname", 15 | "levelno", 16 | "lineno", 17 | "module", 18 | "msecs", 19 | "message", 20 | "msg", 21 | "name", 22 | "pathname", 23 | "process", 24 | "processName", 25 | "relativeCreated", 26 | "stack_info", 27 | "thread", 28 | "threadName", 29 | "taskName", 30 | } 31 | ) 32 | 33 | 34 | class HubcastJSONFormatter(logging.Formatter): 35 | def __init__(self, *, fmt_keys=None): 36 | super().__init__() 37 | self.fmt_keys = fmt_keys or {} 38 | 39 | def format(self, record: logging.LogRecord) -> str: 40 | record_dict = record.__dict__ 41 | 42 | # fields that are only included 43 | log_data = { 44 | "timestamp": dt.datetime.fromtimestamp( 45 | record.created, tz=dt.timezone.utc 46 | ).isoformat(timespec="microseconds"), 47 | "level": record.levelname, 48 | "logger": record.name, 49 | } 50 | 51 | # exclude suppressed messages 52 | if not (record.name == "aiohttp.access" and "message" in self.fmt_keys): 53 | log_data["message"] = record.getMessage() 54 | 55 | # add exception info 56 | if record.exc_info: 57 | log_data["exc_info"] = self.formatException(record.exc_info) 58 | if record.stack_info: 59 | log_data["stack_info"] = self.formatStack(record.stack_info) 60 | 61 | # map additional fields via fmt_keys 62 | for output_key, attr_name in self.fmt_keys.items(): 63 | if output_key in log_data: 64 | continue # already populated 65 | if hasattr(record, attr_name): 66 | log_data[output_key] = getattr(record, attr_name) 67 | 68 | # add any user-defined extra fields (from logger(..., extra={...})) 69 | for key, val in record_dict.items(): 70 | if key not in LOG_RECORD_BUILTIN_ATTRS and key not in log_data: 71 | log_data[key] = val 72 | 73 | return json.dumps(log_data, default=str) 74 | 75 | 76 | class HubcastConsoleFormatter(logging.Formatter): 77 | def format(self, record: logging.LogRecord) -> str: 78 | logger = logging.getLogger(record.name) 79 | show_traceback = logger.isEnabledFor(logging.DEBUG) 80 | 81 | # temporarily suppress traceback if we're not in DEBUG mode 82 | original_exc_info = record.exc_info 83 | if not show_traceback: 84 | record.exc_info = None 85 | 86 | # format the base log line 87 | try: 88 | base = super().format(record) 89 | finally: 90 | # always restore exc_info after formatting 91 | record.exc_info = original_exc_info 92 | 93 | # skip extras for aiohttp.access since they provide redundant info 94 | if record.name == "aiohttp.access": 95 | return base 96 | 97 | # add any user-defined extra fields 98 | extras = { 99 | k: v 100 | for k, v in record.__dict__.items() 101 | if k not in LOG_RECORD_BUILTIN_ATTRS 102 | } 103 | 104 | # format extras 105 | 106 | if extras: 107 | extra_str = " " + " ".join(f"{k}={v}" for k, v in extras.items()) 108 | else: 109 | extra_str = "" 110 | 111 | return base + extra_str 112 | -------------------------------------------------------------------------------- /src/hubcast/clients/github/auth.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Tuple 3 | 4 | import aiohttp 5 | import gidgethub.apps as gha 6 | from gidgethub import aiohttp as gh_aiohttp 7 | 8 | from hubcast.clients.utils import TokenCache 9 | 10 | # location for authenticated app to get a token for one of its installations 11 | # bandit thinks this is a hardcoded password, we ignore security checks on this line 12 | INSTALLATION_TOKEN_URL = "/app/installations/{installation_id}/access_tokens" # nosec B105 13 | 14 | 15 | class GitHubAuthenticator: 16 | """ 17 | An authenticator and token handler for GitHub. 18 | 19 | Attributes: 20 | ---------- 21 | requester: str 22 | A GitHub bot user to act as and perform actions as. 23 | private_key: str 24 | A pem encoded string of the GitHub App's private key 25 | which is used to generate JWTs and other access tokens. 26 | app_id: str 27 | A string of the numeric GitHub App's ID. 28 | """ 29 | 30 | def __init__(self, requester: str, private_key: str, app_id: str) -> None: 31 | self.requester = requester 32 | self.private_key = private_key 33 | self.app_id = app_id 34 | self._tokens = TokenCache() 35 | self._id_dict = {} 36 | 37 | async def get_installation_id(self, owner: str, repo: str) -> str: 38 | if (owner, repo) not in self._id_dict: 39 | async with aiohttp.ClientSession() as session: 40 | gh = gh_aiohttp.GitHubAPI(session, self.requester) 41 | result = await gh.getitem( 42 | f"/repos/{owner}/{repo}/installation", 43 | accept="application/vnd.github+json", 44 | jwt=await self.get_jwt(), 45 | ) 46 | self._id_dict[(owner, repo)] = result["id"] 47 | 48 | return self._id_dict[(owner, repo)] 49 | 50 | async def authenticate_installation(self, owner: str, repo: str) -> str: 51 | """ 52 | Get an installation access token for the application. 53 | Renew the JWT if necessary, then use it to get an installation access 54 | token from github, if necessary. 55 | """ 56 | 57 | installation_id = await self.get_installation_id(owner, repo) 58 | 59 | async def renew_installation_token(): 60 | async with aiohttp.ClientSession() as session: 61 | gh = gh_aiohttp.GitHubAPI(session, self.requester) 62 | 63 | # Use the JWT to get a limited-life OAuth token for a particular 64 | # installation of the app. Note that we get a JWT only when 65 | # necessary -- when we need to renew the installation token. 66 | result = await gh.post( 67 | INSTALLATION_TOKEN_URL, 68 | {"installation_id": installation_id}, 69 | data=b"", 70 | accept="application/vnd.github.machine-man-preview+json", 71 | jwt=await self.get_jwt(), 72 | ) 73 | 74 | expires = self.parse_isotime(result["expires_at"]) 75 | token = result["token"] 76 | return (expires, token) 77 | 78 | return await self._tokens.get(installation_id, renew_installation_token) 79 | 80 | def parse_isotime(self, timestr: str) -> int: 81 | """Convert UTC ISO 8601 time stamp to seconds in epoch""" 82 | if timestr[-1] != "Z": 83 | raise ValueError(f"Time String '{timestr}' not in UTC") 84 | return int(time.mktime(time.strptime(timestr[:-1], "%Y-%m-%dT%H:%M:%S"))) 85 | 86 | async def get_jwt(self) -> str: 87 | """Get a JWT from cache, creating a new one if necessary.""" 88 | 89 | async def renew_jwt() -> Tuple[float, str]: 90 | # GitHub requires that you create a JWT signed with the application's 91 | # private key. You need the app id and the private key, and you can 92 | # use this gidgethub method to create the JWT. 93 | now = time.time() 94 | jwt = gha.get_jwt(app_id=self.app_id, private_key=self.private_key) 95 | 96 | # gidgethub JWT's expire after 10 minutes (you cannot change it) 97 | return (now + 10 * 60), jwt 98 | 99 | return await self._tokens.get("JWT", renew_jwt) 100 | -------------------------------------------------------------------------------- /src/hubcast/clients/gitlab/auth.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta, timezone 2 | from typing import Tuple 3 | 4 | import aiohttp 5 | import gidgetlab.aiohttp 6 | 7 | from hubcast.clients.utils import TokenCache 8 | 9 | TOKEN_NAME = "hubcast-impersonation" # nosec B105 10 | # api scope needed for reading pipelines, setting webhooks 11 | # read_repository and write_repository needed for repo access 12 | GL_SCOPES = ("api", "read_repository", "write_repository") 13 | 14 | 15 | class GitLabAuthenticator: 16 | """ 17 | An authenticator and token handler for GitLab. 18 | 19 | Attributes: 20 | ---------- 21 | instance_url: str 22 | URL of the GitLab instance. 23 | requester: str 24 | A string identifying who is responsible for the requests. 25 | admin_token: str 26 | A personal access token with `api` scope and created by an administrator. 27 | """ 28 | 29 | def __init__(self, instance_url: str, requester: str, admin_token: str): 30 | self.instance_url = instance_url 31 | self.requester = requester 32 | self.admin_token = admin_token 33 | self._tokens = TokenCache() 34 | 35 | async def authenticate_user( 36 | self, 37 | username: str, 38 | scopes: list = GL_SCOPES, 39 | expire_days: int = 1, 40 | ) -> str: 41 | """ 42 | Returns an impersonation token for a user with specified scopes; maintains a cache of previously created tokens. 43 | GitLab does not allow granular expiration times for tokens, so we set expiration in days (defaulting to 1) 44 | 45 | Parameters: 46 | ---------- 47 | username: str 48 | username of the user to impersonate 49 | scopes: list 50 | scopes assigned to any impersonation token created. see gitlab docs for options: 51 | https://docs.gitlab.com/user/profile/personal_access_tokens/#personal-access-token-scopes. 52 | expire_days: int 53 | the number of days after which the token will expire on the gitlab server 54 | """ 55 | 56 | async def renew_impersonation_token(): 57 | # the tokens API requires user IDs, but hubcast's account mapping returns usernames 58 | user_id = await self._get_user_id(username) 59 | 60 | async with aiohttp.ClientSession() as session: 61 | gl = gidgetlab.aiohttp.GitLabAPI( 62 | session, 63 | self.requester, 64 | access_token=self.admin_token, 65 | url=self.instance_url, 66 | ) 67 | 68 | url = f"/users/{user_id}/impersonation_tokens" 69 | expires_day, expires_timestamp = self._date_after_days(expire_days) 70 | 71 | token = await gl.post( 72 | url, 73 | data={ 74 | "user_id": user_id, 75 | "name": TOKEN_NAME, 76 | "description": "Created by Hubcast for CI sync and status reporting.", 77 | "expires_at": expires_day, 78 | "scopes": scopes, 79 | }, 80 | ) 81 | 82 | return (expires_timestamp, token["token"]) 83 | 84 | # the caching key is username so that _get_user_id can be avoided on cache hits 85 | return await self._tokens.get( 86 | f"impersonation:{username}", renew_impersonation_token 87 | ) 88 | 89 | async def _get_user_id(self, username: str) -> int: 90 | """Retrieve the user ID for a given username from the GitLab instance.""" 91 | async with aiohttp.ClientSession() as session: 92 | gl = gidgetlab.aiohttp.GitLabAPI( 93 | session, 94 | self.requester, 95 | access_token=self.admin_token, 96 | url=self.instance_url, 97 | ) 98 | 99 | res = await gl.getitem(f"/users?username={username}") 100 | if not res: 101 | raise ValueError(f"user '{username}' not found on GitLab instance.") 102 | return res[0]["id"] 103 | 104 | @staticmethod 105 | def _date_after_days(days: int) -> Tuple[str, int]: 106 | """Returns UTC date string in YYYY-MM-DD format and midnight UTC timestamp.""" 107 | dt = (datetime.now(timezone.utc) + timedelta(days=days)).replace( 108 | hour=0, minute=0, second=0, microsecond=0 109 | ) 110 | return dt.strftime("%Y-%m-%d"), int(dt.timestamp()) 111 | 112 | 113 | class GitLabSingleUserAuthenticator: 114 | """ 115 | An authenticator for developers and users without admin access to a GitLab instance. 116 | 117 | The token should have the scopes defined in GL_SCOPES for the repos Hubcast will access. 118 | """ 119 | 120 | def __init__(self, access_token: str): 121 | self.access_token = access_token 122 | 123 | async def authenticate_user(self, username: str, *args, **kwargs) -> str: 124 | """Returns the pre-configured token. Keeps the same signature as GitLabAuthenticator for compatibility.""" 125 | return self.access_token 126 | -------------------------------------------------------------------------------- /src/hubcast/clients/gitlab/client.py: -------------------------------------------------------------------------------- 1 | import urllib.parse 2 | from typing import Dict 3 | 4 | import aiohttp 5 | import gidgetlab.aiohttp 6 | 7 | from .auth import GitLabAuthenticator, GitLabSingleUserAuthenticator 8 | 9 | 10 | class GitLabClientFactory: 11 | def __init__( 12 | self, 13 | instance_url: str, 14 | requester: str, 15 | token: str, 16 | callback_url: str, 17 | webhook_secret: str, 18 | token_type: str = "impersonation", # nosec B107 19 | ): 20 | self.requester = requester 21 | 22 | if token_type == "single": # nosec B105 23 | self.auth = GitLabSingleUserAuthenticator(token) 24 | elif token_type == "impersonation": # nosec B105 25 | self.auth = GitLabAuthenticator(instance_url, requester, token) 26 | else: 27 | raise ValueError(f"Unknown GitLab token type: {token_type}") 28 | 29 | self.instance_url = instance_url 30 | self.callback_url = callback_url 31 | self.webhook_secret = webhook_secret 32 | 33 | def create_client(self, user: str): 34 | """creates a GitLabClient for a specific user""" 35 | return GitLabClient( 36 | self.auth, 37 | self.instance_url, 38 | self.callback_url, 39 | self.webhook_secret, 40 | user, 41 | ) 42 | 43 | 44 | class GitLabClient: 45 | def __init__( 46 | self, 47 | auth: GitLabAuthenticator, 48 | instance_url: str, 49 | callback_url: str, 50 | webhook_secret: str, 51 | user: str, 52 | ): 53 | self.auth = auth 54 | self.instance_url = instance_url 55 | self.callback_url = callback_url 56 | self.webhook_secret = webhook_secret 57 | self.user = user 58 | 59 | async def set_webhook(self, gl_fullname: str, data: Dict): 60 | gl_token = await self.auth.authenticate_user(username=self.user) 61 | 62 | new_hook = { 63 | "token": self.webhook_secret, 64 | "url": f"{self.callback_url}?{urllib.parse.urlencode(data)}", 65 | "name": "hubcast", 66 | "description": "Generated by Hubcast for status reporting.", 67 | "job_events": True, 68 | "pipeline_events": True, 69 | "push_events": False, 70 | } 71 | 72 | async with aiohttp.ClientSession() as session: 73 | gl = gidgetlab.aiohttp.GitLabAPI( 74 | session, 75 | requester=self.user, 76 | access_token=gl_token, 77 | url=self.instance_url, 78 | ) 79 | 80 | existing_hook = None 81 | 82 | repo_id = urllib.parse.quote_plus(gl_fullname) 83 | url = f"/projects/{repo_id}/hooks" 84 | 85 | hooks_data = await gl.getitem(url) 86 | for hook in hooks_data: 87 | if hook["name"] == "hubcast": 88 | existing_hook = hook 89 | break 90 | 91 | # if no existing hook is found add one and exit 92 | if not existing_hook: 93 | await gl.post(url, data=new_hook) 94 | return 95 | 96 | # if an existing hook is found compare the values of it 97 | # and the newly generated hook, update the hook to match 98 | # the new configuration if they differ 99 | changed = False 100 | for key in new_hook.keys(): 101 | if key != "token" and existing_hook[key] != new_hook[key]: 102 | changed = True 103 | break 104 | 105 | if changed: 106 | url = f"/projects/{repo_id}/hooks/{existing_hook['id']}" 107 | await gl.put(url, data=new_hook) 108 | 109 | async def get_latest_pipeline(self, gl_fullname: str, ref: str) -> int: 110 | """gets the latest pipeline for an arbitrary GitLab repository and branch. 111 | Returns: 112 | the pipeline's id 113 | """ 114 | 115 | gl_token = await self.auth.authenticate_user(self.user) 116 | 117 | async with aiohttp.ClientSession() as session: 118 | gl = gidgetlab.aiohttp.GitLabAPI( 119 | session, 120 | requester=self.user, 121 | access_token=gl_token, 122 | url=self.instance_url, 123 | ) 124 | 125 | repo_id = urllib.parse.quote_plus(gl_fullname) 126 | 127 | pipeline = await gl.getitem( 128 | f"/projects/{repo_id}/pipelines/latest?ref={ref}" 129 | ) 130 | 131 | return pipeline.get("id") 132 | 133 | async def run_pipeline(self, gl_fullname: str, ref: str) -> str: 134 | """(re)-run a pipeline from an arbitrary GitLab repository and branch. 135 | Returns: 136 | the new pipeline's url 137 | """ 138 | 139 | gl_token = await self.auth.authenticate_user(self.user) 140 | 141 | async with aiohttp.ClientSession() as session: 142 | gl = gidgetlab.aiohttp.GitLabAPI( 143 | session, 144 | requester=self.user, 145 | access_token=gl_token, 146 | url=self.instance_url, 147 | ) 148 | 149 | repo_id = urllib.parse.quote_plus(gl_fullname) 150 | 151 | pipeline = await gl.post(f"/projects/{repo_id}/pipeline?ref={ref}", data={}) 152 | 153 | return pipeline.get("web_url") 154 | 155 | async def retry_pipeline_jobs(self, gl_fullname: str, pipeline_id: int) -> str: 156 | """retries any failed jobs in a pipeline. if there are no jobs that meet this criteria, 157 | calling this has no effect 158 | Returns: 159 | the pipeline's url 160 | """ 161 | 162 | gl_token = await self.auth.authenticate_user(self.user) 163 | 164 | async with aiohttp.ClientSession() as session: 165 | gl = gidgetlab.aiohttp.GitLabAPI( 166 | session, 167 | requester=self.user, 168 | access_token=gl_token, 169 | url=self.instance_url, 170 | ) 171 | 172 | repo_id = urllib.parse.quote_plus(gl_fullname) 173 | 174 | pipeline = await gl.post( 175 | f"/projects/{repo_id}/pipelines/{pipeline_id}/retry", data={} 176 | ) 177 | 178 | return pipeline.get("web_url") 179 | -------------------------------------------------------------------------------- /src/hubcast/clients/github/client.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlparse 2 | 3 | import aiohttp 4 | import yaml 5 | from gidgethub import aiohttp as gh_aiohttp 6 | 7 | from .auth import GitHubAuthenticator 8 | 9 | VALID_GH_REACTIONS = [ 10 | "+1", 11 | "-1", 12 | "laugh", 13 | "hooray", 14 | "confused", 15 | "heart", 16 | "rocket", 17 | "eyes", 18 | ] 19 | 20 | 21 | class InvalidConfigYAMLError(Exception): 22 | pass 23 | 24 | 25 | class GitHubClientFactory: 26 | def __init__(self, app_id, privkey, requester, bot_user): 27 | self.requester = requester 28 | self.auth = GitHubAuthenticator(requester, privkey, app_id) 29 | self.bot_user = bot_user 30 | 31 | def create_client(self, repo_owner, repo_name): 32 | return GitHubClient( 33 | self.auth, self.requester, repo_owner, repo_name, self.bot_user 34 | ) 35 | 36 | 37 | class GitHubClient: 38 | def __init__(self, auth, requester, repo_owner, repo_name, bot_user): 39 | self.auth = auth 40 | self.requester = requester 41 | self.repo_owner = repo_owner 42 | self.repo_name = repo_name 43 | self.bot_user = bot_user 44 | 45 | async def set_check_status( 46 | self, ref: str, check_name: str, status: str, details_url: str 47 | ): 48 | gitlab_netloc = urlparse(details_url).netloc 49 | 50 | # construct upload payload 51 | payload = { 52 | "name": check_name, 53 | "head_sha": ref, 54 | "details_url": details_url, 55 | "output": { 56 | "title": "External Pipeline Run", 57 | "summary": f"[View this pipeline on {gitlab_netloc}]({details_url})", 58 | }, 59 | } 60 | 61 | # for success and failure status write out a conclusion 62 | if status in ("success", "failure", "cancelled"): 63 | payload["status"] = "completed" 64 | payload["conclusion"] = status 65 | else: 66 | payload["status"] = status 67 | 68 | gh_token = await self.auth.authenticate_installation( 69 | self.repo_owner, self.repo_name 70 | ) 71 | 72 | async with aiohttp.ClientSession() as session: 73 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 74 | 75 | # get a list of the checks on a commit 76 | url = f"/repos/{self.repo_owner}/{self.repo_name}/commits/{ref}/check-runs" 77 | data = await gh.getitem(url) 78 | 79 | # search for existing check with GH_CHECK_NAME 80 | existing_check = None 81 | for check in data["check_runs"]: 82 | if check["name"] == check_name: 83 | existing_check = check 84 | break 85 | 86 | # create a new check if no previous check is found, or if the previous 87 | # existing check was marked as completed. (This allows to check re-runs.) 88 | if existing_check is None or existing_check["status"] == "completed": 89 | url = f"/repos/{self.repo_owner}/{self.repo_name}/check-runs" 90 | await gh.post(url, data=payload) 91 | else: 92 | url = f"/repos/{self.repo_owner}/{self.repo_name}/check-runs/{existing_check['id']}" 93 | await gh.patch(url, data=payload) 94 | 95 | async def get_repo_config(self): 96 | gh_token = await self.auth.authenticate_installation( 97 | self.repo_owner, self.repo_name 98 | ) 99 | 100 | async with aiohttp.ClientSession() as session: 101 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 102 | 103 | # get the contents of the repository hubcast.yml file 104 | url = f"/repos/{self.repo_owner}/{self.repo_name}/contents/.github/hubcast.yml" 105 | # get raw contents rather than base64 encoded text 106 | config_str = await gh.getitem(url, accept="application/vnd.github.raw") 107 | 108 | try: 109 | config = yaml.safe_load(config_str) 110 | except yaml.YAMLError: 111 | raise InvalidConfigYAMLError( 112 | f"Failed to parse repo config. repo_owner={self.repo_owner} repo_name={self.repo_name}" 113 | ) 114 | 115 | return config 116 | 117 | async def get_pr(self, id): 118 | """Return individual PR data.""" 119 | gh_token = await self.auth.authenticate_installation( 120 | self.repo_owner, self.repo_name 121 | ) 122 | 123 | async with aiohttp.ClientSession() as session: 124 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 125 | 126 | url = f"/repos/{self.repo_owner}/{self.repo_name}/pulls/{id}" 127 | return await gh.getitem(url) 128 | 129 | async def get_prs(self, branch=None): 130 | """Returns a list of all open PR numbers; can be filtered by internal branches.""" 131 | 132 | gh_token = await self.auth.authenticate_installation( 133 | self.repo_owner, self.repo_name 134 | ) 135 | 136 | async with aiohttp.ClientSession() as session: 137 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 138 | 139 | # https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests 140 | # default is open pull requests 141 | url = f"/repos/{self.repo_owner}/{self.repo_name}/pulls" 142 | if branch: 143 | # head: filter pulls by head user or head organization and branch name 144 | url = f"{url}?head={self.repo_owner}:{branch}" 145 | prs_res = await gh.getitem(url) 146 | return [pr["number"] for pr in prs_res] 147 | 148 | async def post_comment(self, issue_number: int, body: str): 149 | payload = {"body": body} 150 | 151 | gh_token = await self.auth.authenticate_installation( 152 | self.repo_owner, self.repo_name 153 | ) 154 | 155 | async with aiohttp.ClientSession() as session: 156 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 157 | 158 | url = f"/repos/{self.repo_owner}/{self.repo_name}/issues/{issue_number}/comments" 159 | await gh.post(url, data=payload) 160 | 161 | async def react_to_comment(self, comment_id: int, reaction: str): 162 | """Add an emoji reaction to a GitHub PR comment. 163 | See `VALID_GH_REACTIONS` for a list of emoji options. 164 | """ 165 | 166 | if reaction not in VALID_GH_REACTIONS: 167 | raise ValueError(f"{reaction} is not a valid reaction") 168 | 169 | payload = {"content": reaction} 170 | 171 | gh_token = await self.auth.authenticate_installation( 172 | self.repo_owner, self.repo_name 173 | ) 174 | 175 | async with aiohttp.ClientSession() as session: 176 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 177 | 178 | url = f"/repos/{self.repo_owner}/{self.repo_name}/issues/comments/{comment_id}/reactions" 179 | await gh.post(url, data=payload) 180 | 181 | async def get_branch(self, name: str): 182 | """Return individual branch data.""" 183 | 184 | gh_token = await self.auth.authenticate_installation( 185 | self.repo_owner, self.repo_name 186 | ) 187 | 188 | async with aiohttp.ClientSession() as session: 189 | gh = gh_aiohttp.GitHubAPI(session, self.requester, oauth_token=gh_token) 190 | 191 | url = f"/repos/{self.repo_owner}/{self.repo_name}/branches/{name}" 192 | return await gh.getitem(url) 193 | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | ## Installation 3 | > [!WARNING] 4 | > Hubcast is still in development. Easy installation instructions using PyPI 5 | > and production containers coming soon. 6 | 7 | ## Configuration 8 | Hubcast may be configured using environment variables. 9 | 10 | A full set of the current environment variables are shown below. 11 | For development instances of hubcast it is suggested you save these environment 12 | variables in a `.env` file or similar and source it before execution. 13 | 14 | ##### .env 15 | ``` 16 | #------------------------------------------------------------------------ 17 | # Git Repository Settings 18 | #------------------------------------------------------------------------ 19 | # Path for a local Git repository used as an intermediate sync. 20 | export HC_GIT_REPO_PATH="" 21 | 22 | # Remote GitHub Repository. 23 | export HC_GH_REPO="" 24 | 25 | # Remote GitLab Repository. 26 | export HC_GL_REPO="" 27 | 28 | #------------------------------------------------------------------------ 29 | # GitHub Settings 30 | #------------------------------------------------------------------------ 31 | # Name of the GitHub check to be displayed for GitLab Ci Runs. 32 | export HC_GH_CHECK_NAME="" 33 | 34 | # GitHub App Webhook Secret. 35 | export HC_GH_SECRET="" 36 | 37 | # Path to GitHub App's private key. Used for generating tokens. 38 | export HC_GH_PRIVATE_KEY_PATH="" 39 | 40 | # GitHub User to act on the behalf of. 41 | export HC_GH_REQUESTER="" 42 | 43 | # GitHub App Identifier. (Numerical ID.) 44 | export HC_GH_APP_IDENTIFIER= 45 | 46 | #------------------------------------------------------------------------ 47 | # GitLab Settings 48 | #------------------------------------------------------------------------ 49 | # Authentication token for interacting with GitLab. 50 | export HC_GL_ACCESS_TOKEN="" 51 | 52 | # GitLab Webhook Secret. 53 | export HC_GL_SECRET="" 54 | 55 | # GitLab User to act on the behalf of. 56 | export HC_GL_REQUESTER="" 57 | 58 | #------------------------------------------------------------------------ 59 | # Account Map Settings 60 | #------------------------------------------------------------------------ 61 | export HC_ACCOUNT_MAP_PATH="users.yml" 62 | 63 | #------------------------------------------------------------------------ 64 | # General Bot Settings 65 | #------------------------------------------------------------------------ 66 | # Port for hubcast to listen on. 67 | export HC_PORT=3000 68 | ``` 69 | ### Creating a GitHub Repo to Mirror 70 | If you don't already have a GitHub repository you'd like to mirror, you'll 71 | need to create a new repository. (When setting up Hubcast for the first time 72 | it may be a good idea to test it first on a test repository.) 73 | 74 | > [!TIP] 75 | > If you're setting up a local instance of Hubcast to develop on we recommend 76 | > setting up a hubcast-test repository to test out your changes before 77 | > submitting a PR. To create a test repository click 78 | > [here](https://github.com/new?name=hubcast-test). 79 | 80 | 81 | ### Creating a GitHub App 82 | To create a GitHub App you'll need a callback url and a webhook secret. 83 | 84 | If you're deploying Hubcast in production, this callback url be an https link 85 | to your infrastructure. If you're deploying for development, we recommend 86 | using [smee.io](https://smee.io) to forward webhooks to your local environment. 87 | 88 | To see more about contributing to Hubcast check out our contributing 89 | guide [here](contributing.md). 90 | 91 | Your webhook secret should be a shared complex string allowing you to verify 92 | webhooks came from GitHub. To generate a complex secret you can run, 93 | 94 | ```bash 95 | $ openssl rand -base64 24 96 | ``` 97 | 98 | With a callback url and secret handy follow GitHub's 99 | [Registering a GitHub App](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app#registering-a-github-app) 100 | guide to create your app installation. 101 | 102 | We should now be able to fill in the following environment variables in 103 | Hubcast's configuration with the values retrieved from GitHub's setup. 104 | 105 | ##### .env 106 | ``` 107 | #------------------------------------------------------------------------ 108 | # GitHub Settings 109 | #------------------------------------------------------------------------ 110 | # Name of the GitHub check to be displayed for GitLab Ci Runs. 111 | export HC_GH_CHECK_NAME="" 112 | 113 | # GitHub App Webhook Secret. 114 | export HC_GH_SECRET="" 115 | 116 | # Path to GitHub App's private key you downloaded from GitHub. 117 | # Used for generating tokens. 118 | export HC_GH_PRIVATE_KEY_PATH="" 119 | 120 | # GitHub User to act on the behalf of. 121 | export HC_GH_REQUESTER="" 122 | 123 | # GitHub App Identifier. (Numerical ID Shown on GitHub App Config Page.) 124 | export HC_GH_APP_IDENTIFIER= 125 | ``` 126 | 127 | ### Creating a GitLab Repository 128 | If you don't already have a GitLab repository you'd like to mirror into, 129 | we'll need to create one at this point. 130 | 131 | > [!TIP] 132 | > If you're setting up a local instance of Hubcast to develop on we recommend 133 | > setting up a hubcast-test repository to test out your changes before 134 | > submitting a PR. To create a test repository on GitLab.com click 135 | > [here](https://gitlab.com/projects/new). 136 | 137 | ### Creating a GitLab Webhook 138 | Once we've setup a new blank repository in GitLab we can configure it with a 139 | webhook. This webhook will notify Hubcast of in-progress, completed, and failed 140 | CI jobs so those statuses may be passed back to the GitHub Repository. 141 | 142 | Before configuring the webhook in GitLab we'll need to have our callback url handy 143 | and generate another webhoook secret. 144 | 145 | ```bash 146 | $ openssl rand -base64 24 147 | ``` 148 | 149 | One you've got those two values you can follow GitLab's instructions in the 150 | [Webhooks docs](https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#configure-a-webhook-in-gitlab). 151 | 152 | > [!TIP] 153 | > Make sure to set the webhook to trigger on both `job events` and `pipeline events`. 154 | 155 | ### Creating a GitLab Project Access Token 156 | Finally to access the GitLab API we'll need to create Project Access Token 157 | inside the repository by following the instructions 158 | [here](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html#create-a-project-access-token). 159 | 160 | > [!TIP] 161 | > To ensure Hubcast can operate successfully you'll need to give the following permissions 162 | > to the project access token, 163 | > - `read_api` 164 | > - `read_repository` 165 | > - `write_repository` 166 | 167 | ### Completing the GitLab Configuration 168 | After creating the GitLab repository, webhook, and access token we can now fill out the 169 | remainder of the Hubcast GitLab config. 170 | 171 | ##### .env 172 | ```bash 173 | #------------------------------------------------------------------------ 174 | # GitLab Settings 175 | #------------------------------------------------------------------------ 176 | # Authentication token for interacting with GitLab. 177 | export HC_GL_ACCESS_TOKEN="" 178 | 179 | # GitLab Webhook Secret. 180 | export HC_GL_SECRET="" 181 | 182 | # GitLab User to act on the behalf of. 183 | export HC_GL_REQUESTER="" 184 | ``` 185 | 186 | ### Creating a Account Map Document 187 | Lastly we'll need to create a simple account map to map GitHub to GitLab users. 188 | 189 | ##### .env 190 | ```bash 191 | #------------------------------------------------------------------------ 192 | # Account Map Settings 193 | #------------------------------------------------------------------------ 194 | export HC_ACCOUNT_MAP_PATH="users.yml" 195 | ``` 196 | 197 | ##### users.yml 198 | ```yaml 199 | Users: 200 | github_user1: gitlab_user1 201 | github_user2: gitlab_user2 202 | ... 203 | ``` 204 | 205 | ### Finishing Up 206 | At this point we should now be able to launch our instance of hubcast 207 | and have it begin mirroring events from GitHub to GitLab. 208 | 209 | ```bash 210 | $ source .env 211 | $ python -m hubcast 212 | ``` 213 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/hubcast/web/github/routes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from typing import Any 4 | 5 | from gidgethub import routing, sansio 6 | from repligit.asyncio import fetch_pack, ls_remote, send_pack 7 | 8 | from hubcast.web import comments 9 | from hubcast.web.github.utils import get_repo_config 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class GitHubRouter(routing.Router): 15 | """ 16 | Custom router to handle GitHub interactions for hubcast 17 | """ 18 | 19 | async def dispatch(self, event: sansio.Event, *args: Any, **kwargs: Any) -> None: 20 | """Dispatch an event to all registered function(s).""" 21 | found_callbacks = self.fetch(event) 22 | for callback in found_callbacks: 23 | try: 24 | await callback(event, *args, **kwargs) 25 | except Exception: 26 | # this catches errors related to processing of webhook events 27 | log.exception( 28 | "Failed to process GitHub webhook event", 29 | extra={ 30 | "event_type": event.event, 31 | "delivery_id": event.delivery_id, 32 | }, 33 | ) 34 | 35 | 36 | router = GitHubRouter() 37 | 38 | 39 | # ----------------------------------- 40 | # Push Events 41 | # ----------------------------------- 42 | @router.register("push", deleted=False) 43 | async def sync_branch(event, gh, gl, gl_user, *arg, **kwargs): 44 | """Sync the git branch referenced to GitLab.""" 45 | src_repo_url = event.data["repository"]["clone_url"] 46 | src_fullname = event.data["repository"]["full_name"] 47 | src_owner, src_repo_name = src_fullname.split("/") 48 | want_sha = event.data["head_commit"]["id"] 49 | target_ref = event.data["ref"] 50 | 51 | # skip branches from push events that are also pull requests 52 | if await gh.get_prs(branch=target_ref): 53 | return 54 | 55 | repo_config = await get_repo_config(gh, src_fullname, refresh=True) 56 | 57 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 58 | dest_remote_url = f"{gl.instance_url}/{dest_fullname}.git" 59 | 60 | # setup callback webhook on GitLab 61 | webhook_data = { 62 | "gh_owner": src_owner, 63 | "gh_repo": src_repo_name, 64 | "gh_check": repo_config.check_name, 65 | } 66 | await gl.set_webhook(dest_fullname, webhook_data) 67 | 68 | # sync commits from GitHub -> GitLab 69 | gl_refs = await ls_remote(dest_remote_url) 70 | have_shas = set(gl_refs.values()) 71 | from_sha = gl_refs.get(target_ref) or ("0" * 40) 72 | 73 | if want_sha in have_shas: 74 | log.info( 75 | "Target ref already up-to-date", 76 | extra={"repo": src_fullname, "target_ref": target_ref}, 77 | ) 78 | return 79 | 80 | packfile = await fetch_pack( 81 | src_repo_url, 82 | want_sha, 83 | have_shas, 84 | ) 85 | 86 | gl_token = await gl.auth.authenticate_user(gl_user) 87 | 88 | log.info( 89 | "Mirroring refs", 90 | extra={ 91 | "repo": src_fullname, 92 | "from_sha": from_sha, 93 | "want_sha": want_sha, 94 | }, 95 | ) 96 | await send_pack( 97 | dest_remote_url, 98 | target_ref, 99 | from_sha, 100 | want_sha, 101 | packfile, 102 | username=gl_user, 103 | password=gl_token, 104 | ) 105 | 106 | 107 | @router.register("push", deleted=True) 108 | async def remove_branch(event, gh, gl, gl_user, *arg, **kwargs): 109 | src_fullname = event.data["repository"]["full_name"] 110 | target_ref = event.data["ref"] 111 | 112 | repo_config = await get_repo_config(gh, src_fullname, refresh=True) 113 | 114 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 115 | dest_remote_url = f"{gl.instance_url}/{dest_fullname}.git" 116 | 117 | gl_refs = await ls_remote(dest_remote_url) 118 | head_sha = gl_refs.get(target_ref) 119 | null_sha = "0" * 40 120 | 121 | gl_token = await gl.auth.authenticate_user(gl_user) 122 | 123 | log.info("Deleting ref", extra={"repo": src_fullname, "target_ref": target_ref}) 124 | await send_pack( 125 | dest_remote_url, 126 | target_ref, 127 | head_sha, 128 | null_sha, 129 | b"", 130 | username=gl_user, 131 | password=gl_token, 132 | ) 133 | 134 | 135 | # ----------------------------------- 136 | # Pull Request Events 137 | # ----------------------------------- 138 | 139 | 140 | async def sync_pr(pull_request, gh, gl, gl_user): 141 | """Sync the git fork/branch referenced in a PR to GitLab. 142 | 143 | This isn't technically an event handler, but is used a couple different ways in this file. 144 | """ 145 | pull_request_id = pull_request["number"] 146 | 147 | src_repo_url = pull_request["head"]["repo"]["clone_url"] 148 | src_fullname = pull_request["head"]["repo"]["full_name"] 149 | want_sha = pull_request["head"]["sha"] 150 | 151 | # pull requests coming from forks are pushed as branches in the form of 152 | # pr- instead of as their branch name as conflicts could occur 153 | # between multiple repositories 154 | is_pull_request_fork = src_fullname != pull_request["base"]["repo"]["full_name"] 155 | if is_pull_request_fork: 156 | target_ref = f"refs/heads/pr-{pull_request_id}" 157 | else: 158 | target_ref = f"refs/heads/{pull_request['head']['ref']}" 159 | 160 | # get the repository configuration from .github/hubcast.yml 161 | repo_config = await get_repo_config(gh, src_fullname) 162 | 163 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 164 | dest_remote_url = f"{gl.instance_url}/{dest_fullname}.git" 165 | 166 | gl_refs = await ls_remote(dest_remote_url) 167 | have_shas = set(gl_refs.values()) 168 | from_sha = gl_refs.get(target_ref) or ("0" * 40) 169 | 170 | if want_sha in have_shas: 171 | log.info( 172 | "Target ref already up-to-date", 173 | extra={"repo": src_fullname, "target_ref": target_ref}, 174 | ) 175 | return 176 | 177 | # fetch differential packfile with all new commits 178 | packfile = await fetch_pack( 179 | src_repo_url, 180 | want_sha, 181 | have_shas, 182 | ) 183 | 184 | gl_token = await gl.auth.authenticate_user(gl_user) 185 | 186 | # upload packfile to gitlab repository 187 | log.info( 188 | "Mirroring refs", 189 | extra={ 190 | "repo": src_fullname, 191 | "from_sha": from_sha, 192 | "want_sha": want_sha, 193 | }, 194 | ) 195 | await send_pack( 196 | dest_remote_url, 197 | target_ref, 198 | from_sha, 199 | want_sha, 200 | packfile, 201 | username=gl_user, 202 | password=gl_token, 203 | ) 204 | 205 | 206 | @router.register("pull_request", action="opened") 207 | @router.register("pull_request", action="reopened") 208 | @router.register("pull_request", action="synchronize") 209 | async def sync_pr_event(event, gh, gl, gl_user, *arg, **kwargs): 210 | """Sync the git fork/branch referenced in a PR to GitLab.""" 211 | pull_request = event.data["pull_request"] 212 | await sync_pr(pull_request, gh, gl, gl_user) 213 | 214 | 215 | @router.register("pull_request", action="closed") 216 | async def remove_pr(event, gh, gl, gl_user, *arg, **kwargs): 217 | pull_request = event.data["pull_request"] 218 | src_fullname = pull_request["head"]["repo"]["full_name"] 219 | 220 | # if the pull request comes from a fork we should clean up 221 | # the branch upon closing or merging the PR. However, if the 222 | # pull request comes from an internal branch we should wait 223 | # to clean up the branch when the branch is deleted from the 224 | # internal repository 225 | is_pull_request_fork = src_fullname != pull_request["base"]["repo"]["full_name"] 226 | if not is_pull_request_fork: 227 | return 228 | 229 | pull_request_id = pull_request["number"] 230 | target_ref = f"refs/heads/pr-{pull_request_id}" 231 | 232 | # get the repository configuration from .github/hubcast.yml 233 | repo_config = await get_repo_config(gh, src_fullname) 234 | 235 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 236 | dest_remote_url = f"{gl.instance_url}/{dest_fullname}.git" 237 | 238 | gl_refs = await ls_remote(dest_remote_url) 239 | head_sha = gl_refs.get(target_ref) 240 | null_sha = "0" * 40 241 | 242 | gl_token = await gl.auth.authenticate_user(gl_user) 243 | 244 | log.info("Deleting ref", extra={"repo": src_fullname, "target_ref": target_ref}) 245 | await send_pack( 246 | dest_remote_url, 247 | target_ref, 248 | head_sha, 249 | null_sha, 250 | b"", 251 | username=gl_user, 252 | password=gl_token, 253 | ) 254 | 255 | 256 | @router.register("issue_comment", action="created") 257 | async def respond_comment(event, gh, gl, gl_user, *arg, **kwargs): 258 | # differentiate issue vs PR comment 259 | if "pull_request" not in event.data["issue"]: 260 | return 261 | 262 | comment = event.data["comment"]["body"] 263 | response = None 264 | plus_one = False 265 | 266 | if re.search(f"@{gh.bot_user} help", comment, re.IGNORECASE): 267 | response = comments.help_message(gh.bot_user) 268 | 269 | elif re.search(f"@{gh.bot_user} approve", comment, re.IGNORECASE): 270 | # syncs PR changes to the destination on behalf of the commenter 271 | # this does not handle PR deletions, those will need to be manually cleaned by project maintainers 272 | pull_request_id = event.data["issue"]["number"] 273 | pull_request = await gh.get_pr(pull_request_id) 274 | await sync_pr(pull_request, gh, gl, gl_user) 275 | 276 | # note: the user will see a +1 regardless of whether a sync truly occurred 277 | plus_one = True 278 | 279 | elif re.search( 280 | f"@{gh.bot_user} (re[-]?)?(run|start) pipeline", comment, re.IGNORECASE 281 | ): 282 | pull_request_id = event.data["issue"]["number"] 283 | pull_request = await gh.get_pr(pull_request_id) 284 | # sync the PR in case it fell out of sync 285 | await sync_pr(pull_request, gh, gl, gl_user) 286 | 287 | # get the branch this PR belongs to 288 | src_fullname = pull_request["head"]["repo"]["full_name"] 289 | # pull requests coming from forks are pushed as branches in the form of 290 | # pr- instead of as their branch name as conflicts could occur 291 | # between multiple repositories 292 | is_pull_request_fork = src_fullname != pull_request["base"]["repo"]["full_name"] 293 | if is_pull_request_fork: 294 | branch = f"pr-{pull_request_id}" 295 | else: 296 | branch = pull_request["head"]["ref"] 297 | 298 | # get the gitlab repo information and run the pipeline 299 | repo_config = await get_repo_config(gh, src_fullname, refresh=True) 300 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 301 | pipeline_url = await gl.run_pipeline(dest_fullname, branch) 302 | 303 | if pipeline_url: 304 | response = f"I've started a new [pipeline]({pipeline_url}) for you!" 305 | plus_one = True 306 | else: 307 | response = "I had a problem starting the pipeline." 308 | 309 | elif re.search( 310 | f"@{gh.bot_user} restart failed(?:[- ]?jobs)?", comment, re.IGNORECASE 311 | ): 312 | pull_request_id = event.data["issue"]["number"] 313 | pull_request = await gh.get_pr(pull_request_id) 314 | # if a pipeline failed, we give the user the option to restart any failed jobs 315 | # we don't want to re-sync the branch, as a new pipeline would be created 316 | # and would defeat the purpose of individually restarting failed jobs 317 | 318 | # get the branch this PR belongs to 319 | src_fullname = pull_request["head"]["repo"]["full_name"] 320 | # pull requests coming from forks are pushed as branches in the form of 321 | # pr- instead of as their branch name as conflicts could occur 322 | # between multiple repositories 323 | is_pull_request_fork = src_fullname != pull_request["base"]["repo"]["full_name"] 324 | if is_pull_request_fork: 325 | branch = f"pr-{pull_request_id}" 326 | else: 327 | branch = pull_request["head"]["ref"] 328 | 329 | # get the gitlab repo information and run the pipeline 330 | repo_config = await get_repo_config(gh, src_fullname, refresh=True) 331 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 332 | pipeline_id = await gl.get_latest_pipeline(dest_fullname, branch) 333 | 334 | if pipeline_id: 335 | pipeline_url = await gl.retry_pipeline_jobs(dest_fullname, pipeline_id) 336 | 337 | if pipeline_url: 338 | response = ( 339 | f"I've retried any failed jobs in the [pipeline]({pipeline_url})!" 340 | ) 341 | plus_one = True 342 | else: 343 | response = "I had a problem retrying jobs in the pipeline." 344 | else: 345 | response = "No pipeline exists." 346 | 347 | if response: 348 | await gh.post_comment(event.data["issue"]["number"], response) 349 | 350 | if plus_one: 351 | await gh.react_to_comment(event.data["comment"]["id"], "+1") 352 | 353 | 354 | @router.register("check_run", action="rerequested") 355 | async def rerun_check(event, gh, gl, gl_user, *arg, **kwargs): 356 | """ 357 | Handles a user re-running a check run for the latest commit in the branch. 358 | See https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=rerequested#check_run. 359 | """ 360 | src_fullname = event.data["repository"]["full_name"] 361 | branch = event.data["check_run"]["check_suite"]["head_branch"] 362 | check_run_commit = event.data["check_run"]["head_sha"] 363 | 364 | # get the latest commit on the branch from GH 365 | branch_data = await gh.get_branch(branch) 366 | latest_commit = branch_data["commit"]["sha"] 367 | 368 | # only rerun if this commit is the head of the branch 369 | if check_run_commit != latest_commit: 370 | log.info("user tried to re-run check for old commit") 371 | return 372 | 373 | # get the GL repo info and run the pipeline 374 | repo_config = await get_repo_config(gh, src_fullname, refresh=True) 375 | dest_fullname = f"{repo_config.dest_org}/{repo_config.dest_name}" 376 | await gl.run_pipeline(dest_fullname, branch) 377 | -------------------------------------------------------------------------------- /logo/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 291 | --------------------------------------------------------------------------------