├── .docs-requirements.txt ├── octomachinery ├── app │ ├── runtime │ │ ├── __init__.py │ │ ├── context.py │ │ ├── config.py │ │ ├── utils.py │ │ └── installation_utils.py │ ├── __init__.py │ ├── routing │ │ ├── abc.py │ │ ├── decorators.py │ │ ├── routers.py │ │ ├── __init__.py │ │ └── webhooks_dispatcher.py │ ├── server │ │ ├── config.py │ │ ├── runner.py │ │ └── machinery.py │ ├── action │ │ ├── config.py │ │ └── runner.py │ └── config.py ├── __init__.py ├── routing │ ├── __init__.py │ ├── decorators.py │ ├── default_router.py │ ├── abc.py │ ├── routers.py │ └── webhooks_dispatcher.py ├── github │ ├── errors │ │ └── __init__.py │ ├── models │ │ ├── _compat.py │ │ ├── utils.py │ │ ├── action_outcomes.py │ │ ├── __init__.py │ │ ├── private_key.py │ │ ├── checks_api_requests.py │ │ └── events.py │ ├── api │ │ ├── tokens.py │ │ ├── utils.py │ │ ├── raw_client.py │ │ └── app_client.py │ ├── entities │ │ ├── action.py │ │ └── app_installation.py │ ├── config │ │ └── app.py │ └── utils │ │ └── event_utils.py ├── runtime │ ├── context.py │ └── utils.py ├── utils │ ├── versiontools.py │ └── asynctools.py └── cli │ └── __main__.py ├── .github ├── deadpendency.yaml └── FUNDING.yml ├── .git_archival.txt ├── pyproject.toml ├── docs ├── _templates │ ├── project-description.html │ └── github-sponsors.html ├── index.rst ├── getting-started.rst ├── _static │ └── custom.css ├── howto-guides.rst └── conf.py ├── .gitattributes ├── .yamllint ├── mypy.ini ├── .readthedocs.yml ├── .isort.cfg ├── tests ├── utils │ ├── versiontools_test.py │ └── asynctools_test.py ├── conftest.py ├── app │ ├── routing │ │ └── decorators_test.py │ ├── action │ │ └── runner_test.py │ └── server │ │ └── machinery_test.py ├── github │ ├── models │ │ ├── private_key_test.py │ │ └── utils_test.py │ └── utils │ │ └── event_utils_test.py └── circular_imports_test.py ├── .gitignore ├── pytest.ini ├── setup.cfg ├── .git-blame-ignore-revs ├── README.rst ├── .flake8 ├── .pre-commit-config.yaml └── tox.ini /.docs-requirements.txt: -------------------------------------------------------------------------------- 1 | pip >= 19.0.3 2 | setuptools >= 40.9.0 3 | -------------------------------------------------------------------------------- /octomachinery/app/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | """GitHub App runtime management.""" 2 | -------------------------------------------------------------------------------- /octomachinery/__init__.py: -------------------------------------------------------------------------------- 1 | """Octomachinery framework for developing GitHub Apps and Actions.""" 2 | -------------------------------------------------------------------------------- /.github/deadpendency.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignore-failures: 3 | python: 4 | - setuptools_scm_git_archive 5 | ... 6 | -------------------------------------------------------------------------------- /octomachinery/app/__init__.py: -------------------------------------------------------------------------------- 1 | """GitHub App infra.""" 2 | 3 | from .server.runner import run # noqa: F401 4 | -------------------------------------------------------------------------------- /.git_archival.txt: -------------------------------------------------------------------------------- 1 | node: 09b5e5ca2f7eb1f7d7b6f974392456a25411a11a 2 | node-date: 2024-08-26T12:05:57+02:00 3 | describe-name: v0.3.11-13-g09b5e5c 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 68", 4 | "setuptools_scm [toml] >= 7.1.0", 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | 8 | [tool.setuptools_scm] 9 | -------------------------------------------------------------------------------- /octomachinery/app/routing/abc.py: -------------------------------------------------------------------------------- 1 | """Octomachinery router base interface definitions proxy.""" 2 | 3 | # pylint: disable=unused-import 4 | from ...routing.abc import OctomachineryRouterBase # noqa: F401 5 | -------------------------------------------------------------------------------- /octomachinery/app/runtime/context.py: -------------------------------------------------------------------------------- 1 | """The globally accessible context vars mapping proxy.""" 2 | 3 | # pylint: disable=unused-import 4 | from ...runtime.context import RUNTIME_CONTEXT # noqa: F401 5 | -------------------------------------------------------------------------------- /octomachinery/app/routing/decorators.py: -------------------------------------------------------------------------------- 1 | """Webhook event processing helper decorators proxy.""" 2 | 3 | # pylint: disable=unused-import 4 | from ...routing.decorators import process_webhook_payload # noqa: F401 5 | -------------------------------------------------------------------------------- /docs/_templates/project-description.html: -------------------------------------------------------------------------------- 1 |

{{ theme_prj_summary }}

9 | -------------------------------------------------------------------------------- /octomachinery/routing/__init__.py: -------------------------------------------------------------------------------- 1 | """GitHub webhooks routing.""" 2 | 3 | 4 | # pylint: disable=unused-import 5 | from .default_router import ( # noqa: F401 6 | WEBHOOK_EVENTS_ROUTER, dispatch_event, process_event, 7 | process_event_actions, 8 | ) 9 | -------------------------------------------------------------------------------- /octomachinery/app/routing/routers.py: -------------------------------------------------------------------------------- 1 | """Octomachinery event dispatchers collection proxy.""" 2 | 3 | # pylint: disable=unused-import 4 | from ...routing.routers import ( # noqa: F401 5 | ConcurrentRouter, GidgetHubRouterBase, NonBlockingConcurrentRouter, 6 | ) 7 | -------------------------------------------------------------------------------- /octomachinery/app/routing/__init__.py: -------------------------------------------------------------------------------- 1 | """GitHub webhooks routing proxy.""" 2 | 3 | # pylint: disable=unused-import 4 | from ...routing.default_router import ( # noqa: F401 5 | WEBHOOK_EVENTS_ROUTER, dispatch_event, process_event, 6 | process_event_actions, 7 | ) 8 | -------------------------------------------------------------------------------- /docs/_templates/github-sponsors.html: -------------------------------------------------------------------------------- 1 | {# TODO: Reimplement this w/o an iframe and make it dark mode ready #} 2 | 9 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Force LF line endings for text files 2 | * text=auto eol=lf 3 | 4 | # Needed for setuptools-scm-git-archive 5 | .git_archival.txt export-subst 6 | 7 | # Blame ignore list entries are expected to always be appended, never edited 8 | .git-blame-ignore-revs merge=union 9 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | extends: default 4 | 5 | rules: 6 | indentation: 7 | level: error 8 | indent-sequences: false 9 | truthy: 10 | allowed-values: 11 | - >- 12 | false 13 | - >- 14 | true 15 | - >- # Allow "on" key name in GHA CI/CD workflow definitions 16 | on 17 | 18 | ... 19 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | ignore_missing_imports = True 3 | 4 | check_untyped_defs = True 5 | 6 | disallow_any_generics = True 7 | # disallow_untyped_defs = True 8 | 9 | enable_error_code = 10 | ignore-without-code 11 | 12 | follow_imports = normal 13 | 14 | strict_optional = True 15 | 16 | warn_redundant_casts = True 17 | warn_unused_ignores = True 18 | -------------------------------------------------------------------------------- /octomachinery/app/server/config.py: -------------------------------------------------------------------------------- 1 | """The web-server configuration.""" 2 | 3 | import environ 4 | 5 | 6 | @environ.config 7 | class WebServerConfig: # pylint: disable=too-few-public-methods 8 | """Config of a web-server.""" 9 | 10 | host = environ.var('0.0.0.0', name='HOST') 11 | port = environ.var(8080, name='PORT', converter=int) 12 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | version: 2 4 | 5 | formats: all 6 | 7 | build: 8 | os: "ubuntu-22.04" 9 | tools: 10 | python: "3.7" 11 | 12 | python: 13 | install: 14 | # - requirements: .docs-requirements.txt 15 | # - method: pip 16 | # path: pip >= 19.0.3 17 | - method: pip 18 | path: . 19 | extra_requirements: 20 | - docs 21 | 22 | ... 23 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | custom: 4 | - https://www.comebackalive.in.ua/donate 5 | - https://github.com/vshymanskyy/StandWithUkraine#for-maintainers-and-authors 6 | - https://www.paypal.me/webknjazCZ 7 | - https://webknjaz.me 8 | 9 | github: 10 | - webknjaz 11 | 12 | ko_fi: webknjaz 13 | 14 | liberapay: webknjaz 15 | 16 | tidelift: pypi/octomachinery 17 | 18 | ... 19 | -------------------------------------------------------------------------------- /octomachinery/github/errors/__init__.py: -------------------------------------------------------------------------------- 1 | """Exceptions collection related to interactions with GitHub.""" 2 | 3 | from gidgethub import GitHubException 4 | 5 | import attr 6 | 7 | # pylint: disable=relative-beyond-top-level 8 | from ..models.action_outcomes import ActionOutcome 9 | 10 | 11 | class GitHubError(GitHubException): 12 | """Generic GitHub-related error.""" 13 | 14 | 15 | @attr.dataclass 16 | class GitHubActionError(GitHubError): 17 | """Generic GitHub-related error.""" 18 | 19 | _outcome: ActionOutcome 20 | 21 | def terminate_action(self): 22 | """Terminate current process using corresponding return code.""" 23 | self._outcome.raise_it() 24 | -------------------------------------------------------------------------------- /octomachinery/routing/decorators.py: -------------------------------------------------------------------------------- 1 | """Webhook event processing helper decorators.""" 2 | 3 | from __future__ import annotations 4 | 5 | from functools import wraps 6 | from typing import TYPE_CHECKING, Any 7 | 8 | 9 | if TYPE_CHECKING: 10 | # pylint: disable=relative-beyond-top-level 11 | from ..github.models.events import GitHubEvent 12 | 13 | 14 | __all__ = ('process_webhook_payload',) 15 | 16 | 17 | def process_webhook_payload(wrapped_function): 18 | """Bypass event object keys-values as args to the handler.""" 19 | @wraps(wrapped_function) 20 | def wrapper(event: GitHubEvent) -> Any: 21 | return wrapped_function(**event.payload) 22 | return wrapper 23 | -------------------------------------------------------------------------------- /octomachinery/runtime/context.py: -------------------------------------------------------------------------------- 1 | """The globally accessible context vars mapping. 2 | 3 | It is supposed to be used as follows: 4 | 5 | >>> from octomachinery.runtime.context import RUNTIME_CONTEXT 6 | 7 | Or shorter: 8 | >>> from octomachinery import RUNTIME_CONTEXT 9 | """ 10 | # pylint: disable=relative-beyond-top-level 11 | from .utils import _ContextMap 12 | 13 | 14 | RUNTIME_CONTEXT = _ContextMap( 15 | app_installation='app installation', 16 | app_installation_client='app installation client', 17 | config='config context', 18 | github_app='github app', 19 | github_event='GitHub Event', 20 | IS_GITHUB_ACTION='Is GitHub Action', 21 | IS_GITHUB_APP='Is GitHub App', 22 | ) 23 | -------------------------------------------------------------------------------- /octomachinery/app/runtime/config.py: -------------------------------------------------------------------------------- 1 | """The application runtime configuration.""" 2 | import attr 3 | import environ 4 | 5 | from .utils import detect_env_mode 6 | 7 | 8 | @environ.config 9 | class RuntimeConfig: # pylint: disable=too-few-public-methods 10 | """Config of runtime env.""" 11 | 12 | debug = environ.bool_var(False, name='DEBUG') 13 | env = environ.var( 14 | 'prod', name='ENV', 15 | validator=attr.validators.in_(('dev', 'prod')), 16 | ) 17 | mode = environ.var( 18 | 'auto', name='OCTOMACHINERY_APP_MODE', 19 | converter=lambda val: detect_env_mode() if val == 'auto' else val, 20 | validator=attr.validators.in_(('app', 'action')), 21 | ) 22 | -------------------------------------------------------------------------------- /octomachinery/github/models/_compat.py: -------------------------------------------------------------------------------- 1 | """Compatibility shims for the models subpackage.""" 2 | from functools import wraps as _wraps_function 3 | 4 | from jwt import encode as _compute_jwt 5 | 6 | 7 | try: 8 | from jwt import __version__ as _pyjwt_version_str 9 | except ImportError: 10 | _pyjwt_version_str = '0.0.0' 11 | 12 | 13 | _pyjwt_version_info = tuple(map(int, _pyjwt_version_str.split('.'))) 14 | _is_pyjwt_above_v2_0 = _pyjwt_version_info >= (2, 0, 0) 15 | 16 | 17 | @_wraps_function(_compute_jwt) 18 | def _compute_jwt_below_v2_0(*args, **kwargs) -> str: 19 | return _compute_jwt(*args, **kwargs).decode('utf-8') 20 | 21 | 22 | compute_jwt = _compute_jwt if _is_pyjwt_above_v2_0 else _compute_jwt_below_v2_0 23 | -------------------------------------------------------------------------------- /octomachinery/utils/versiontools.py: -------------------------------------------------------------------------------- 1 | """Version tools set.""" 2 | 3 | from typing import Callable, Optional, Union 4 | 5 | from setuptools_scm import get_version 6 | from setuptools_scm.version import ScmVersion 7 | 8 | 9 | def get_version_from_scm_tag( 10 | *, 11 | root: str = '.', 12 | relative_to: Optional[str] = None, 13 | local_scheme: Union[ 14 | Callable[[ScmVersion], str], str, 15 | ] = 'node-and-date', 16 | ) -> str: 17 | """Retrieve the version from SCM tag in Git or Hg.""" 18 | try: 19 | return get_version( 20 | root=root, 21 | relative_to=relative_to, 22 | local_scheme=local_scheme, 23 | ) 24 | except LookupError: 25 | return 'unknown' 26 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | # https://github.com/timothycrosley/isort/wiki/isort-Settings 2 | [settings] 3 | default_section = THIRDPARTY 4 | # force_to_top = file1.py,file2.py 5 | # forced_separate = django.contrib,django.utils 6 | include_trailing_comma = true 7 | indent = 4 8 | known_first_party = octomachinery 9 | # known_future_library = future,pies 10 | # known_standard_library = std,std2 11 | known_testing = pytest, unittest 12 | known_framework = aiohttp, anyio, gidgethub, multidict, yarl 13 | # known_third_party = Cython 14 | # length_sort = 1 15 | # Should be: 80 - 1 16 | line_length = 79 17 | lines_after_imports = 2 18 | # https://github.com/timothycrosley/isort#multi-line-output-modes 19 | multi_line_output = 5 20 | no_lines_before = LOCALFOLDER 21 | sections = FUTURE, STDLIB, FRAMEWORK, TESTING, THIRDPARTY, FIRSTPARTY, LOCALFOLDER 22 | # skip=file3.py,file4.py 23 | use_parentheses = true 24 | verbose = true 25 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. octomachinery documentation master file, created by 2 | sphinx-quickstart on Tue Mar 12 14:26:59 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | |project|: the octobots engine 7 | ============================== 8 | 9 | .. include:: ../README.rst 10 | :end-before: DO-NOT-REMOVE-docs-badges-END 11 | 12 | .. include:: ../README.rst 13 | :start-after: DO-NOT-REMOVE-docs-intro-START 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | :caption: Contents: 18 | 19 | getting-started 20 | howto-guides 21 | 22 | .. toctree:: 23 | :caption: Extras: 24 | 25 | Create a GitHub bot 🤖 26 | octomachinery [www] 🤖 27 | 28 | 29 | 30 | Indices and tables 31 | ================== 32 | 33 | * :ref:`genindex` 34 | * :ref:`modindex` 35 | * :ref:`search` 36 | -------------------------------------------------------------------------------- /octomachinery/app/runtime/utils.py: -------------------------------------------------------------------------------- 1 | """GitHub App runtime context helpers.""" 2 | 3 | import logging 4 | import os 5 | 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def detect_env_mode(): 11 | """Figure out whether we're under GitHub Action environment.""" 12 | for var_suffix in ( 13 | 'WORKFLOW', 14 | 'ACTION', 'ACTOR', 15 | 'REPOSITORY', 16 | 'EVENT_NAME', 'EVENT_PATH', 17 | 'WORKSPACE', 18 | 'SHA', 'REF', 19 | 'TOKEN', 20 | ): 21 | if f'GITHUB_{var_suffix}' not in os.environ: 22 | logger.info( 23 | 'Detected GitHub App mode since ' 24 | 'GITHUB_%s is missing from the env', 25 | var_suffix, 26 | ) 27 | return 'app' 28 | logger.info( 29 | 'Detected GitHub Action mode since all the ' 30 | 'typical env vars are present in the env', 31 | ) 32 | return 'action' 33 | -------------------------------------------------------------------------------- /tests/utils/versiontools_test.py: -------------------------------------------------------------------------------- 1 | """Test suite for version utility helper functions.""" 2 | 3 | import contextlib 4 | import os 5 | import pathlib 6 | import tempfile 7 | 8 | import pytest 9 | 10 | from octomachinery.utils.versiontools import get_version_from_scm_tag 11 | 12 | 13 | @contextlib.contextmanager 14 | def working_directory(path): 15 | """Change working directory and return back to previous on exit.""" 16 | prev_cwd = pathlib.Path.cwd() 17 | os.chdir(path) 18 | try: 19 | yield path 20 | finally: 21 | os.chdir(prev_cwd) 22 | 23 | 24 | @pytest.fixture 25 | def temporary_working_directory(): 26 | """Create a temporary working directory, cd there and back on exit.""" 27 | with tempfile.TemporaryDirectory() as tmp_git_repo_dir: 28 | with working_directory(tmp_git_repo_dir) as path: 29 | yield path 30 | 31 | 32 | def test_get_version_from_scm_tag_outside_git_repo( 33 | temporary_working_directory, 34 | ): 35 | """Check that version is unknown outside of Git repo.""" 36 | assert get_version_from_scm_tag() == 'unknown' 37 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Shared fixtures for tests.""" 2 | import pytest 3 | 4 | from cryptography.hazmat.backends import default_backend 5 | from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key 6 | from cryptography.hazmat.primitives.serialization import ( 7 | Encoding, NoEncryption, PrivateFormat, 8 | ) 9 | 10 | 11 | @pytest.fixture 12 | def rsa_private_key(): 13 | """Generate an RSA private key.""" 14 | return generate_private_key( 15 | public_exponent=65537, 16 | key_size=4096, 17 | backend=default_backend(), 18 | ) 19 | 20 | 21 | @pytest.fixture 22 | def rsa_private_key_bytes(rsa_private_key) -> bytes: 23 | r"""Generate an unencrypted PKCS#1 formatted RSA private key. 24 | 25 | Encoded as PEM-bytes. 26 | 27 | This is what the GitHub-downloaded PEM files contain. 28 | 29 | Ref: https://developer.github.com/apps/building-github-apps/\ 30 | authenticating-with-github-apps/ 31 | """ 32 | return rsa_private_key.private_bytes( 33 | encoding=Encoding.PEM, 34 | format=PrivateFormat.TraditionalOpenSSL, # A.K.A. PKCS#1 35 | encryption_algorithm=NoEncryption(), 36 | ) 37 | -------------------------------------------------------------------------------- /docs/getting-started.rst: -------------------------------------------------------------------------------- 1 | Getting started 2 | =============== 3 | 4 | The documentation isn't ready yet so I suggest you going through the 5 | :doc:`How-to create a GitHub Bot tutorial ` 6 | which should give you basic understanding of GitHub Apps and how to 7 | write them with octomachinery. 8 | 9 | Runtime pre-requisites 10 | ---------------------- 11 | 12 | * Python 3.7+ as octomachinery relies on :py:mod:`contextvars` which 13 | doesn't have a backport. 14 | * GitHub App credentials and GitHub Action events are supplied via 15 | environment variables. They are also loaded from a ``.env`` file if it 16 | exists in a development environment. 17 | 18 | .. warning:: 19 | 20 | Be aware that some tools in your environment may conflict with 21 | autoloading vars from a ``.env`` file. It is recommended to disable 22 | those. One example of such tool is `Pipenv`_. 23 | 24 | .. _`Pipenv`: 25 | https://pipenv.readthedocs.io/en/latest/advanced/ 26 | #automatic-loading-of-env 27 | 28 | For the production deployments, please use a way of supplying env vars 29 | via tools provided by the application orchestration software of your 30 | choice. 31 | -------------------------------------------------------------------------------- /octomachinery/routing/default_router.py: -------------------------------------------------------------------------------- 1 | """Default GitHub event dispatcher.""" 2 | 3 | from functools import wraps 4 | 5 | from .routers import ConcurrentRouter 6 | 7 | 8 | __all__ = ( 9 | 'dispatch_event', 10 | 'process_event', 11 | 'process_event_actions', 12 | 'WEBHOOK_EVENTS_ROUTER', 13 | ) 14 | 15 | 16 | WEBHOOK_EVENTS_ROUTER = ConcurrentRouter() 17 | """An event dispatcher for webhooks.""" 18 | 19 | 20 | dispatch_event = WEBHOOK_EVENTS_ROUTER.dispatch # pylint: disable=invalid-name 21 | process_event = WEBHOOK_EVENTS_ROUTER.register # pylint: disable=invalid-name 22 | 23 | 24 | def process_event_actions(event_name, actions=None): 25 | """Subscribe to multiple events.""" 26 | if actions is None: 27 | actions = [] 28 | 29 | def decorator(original_function): 30 | 31 | def wrapper(*args, **kwargs): 32 | return original_function(*args, **kwargs) 33 | 34 | if not actions: 35 | wrapper = process_event(event_name)(wrapper) 36 | 37 | for action in actions: 38 | wrapper = process_event(event_name, action=action)(wrapper) 39 | 40 | return wraps(original_function)(wrapper) 41 | 42 | return decorator 43 | -------------------------------------------------------------------------------- /octomachinery/routing/abc.py: -------------------------------------------------------------------------------- 1 | """Octomachinery router base interface definitions.""" 2 | 3 | from abc import ABCMeta, abstractmethod 4 | from typing import TYPE_CHECKING, Any, Iterator 5 | 6 | from gidgethub.routing import AsyncCallback 7 | 8 | 9 | if TYPE_CHECKING: 10 | from ..github.models.events import GitHubEvent 11 | 12 | 13 | __all__ = ('OctomachineryRouterBase',) 14 | 15 | 16 | class OctomachineryRouterBase(metaclass=ABCMeta): 17 | """Octomachinery router base.""" 18 | 19 | # pylint: disable=unused-argument 20 | @abstractmethod 21 | def emit_routes_for( 22 | self, event_name: str, event_payload: Any, 23 | ) -> Iterator[AsyncCallback]: 24 | """Emit callbacks that match given event and payload. 25 | 26 | :param str event_name: name of the GitHub event 27 | :param str event_payload: details of the GitHub event 28 | 29 | :yields: coroutine event handlers 30 | """ 31 | 32 | # pylint: disable=unused-argument 33 | @abstractmethod 34 | async def dispatch( 35 | self, event: 'GitHubEvent', 36 | *args: Any, **kwargs: Any, 37 | ) -> None: 38 | """Invoke coroutine handler tasks for the given event.""" 39 | -------------------------------------------------------------------------------- /octomachinery/app/action/config.py: -------------------------------------------------------------------------------- 1 | """GitHub Action environment and metadata representation.""" 2 | 3 | from pathlib import Path 4 | 5 | import environ 6 | 7 | # pylint: disable=relative-beyond-top-level 8 | from ...github.models.utils import SecretStr 9 | 10 | 11 | @environ.config 12 | class GitHubActionConfig: # pylint: disable=too-few-public-methods 13 | """GitHub Action config.""" 14 | 15 | workflow = environ.var( 16 | None, name='GITHUB_WORKFLOW', 17 | ) 18 | action = environ.var( 19 | None, name='GITHUB_ACTION', 20 | ) 21 | actor = environ.var( 22 | None, name='GITHUB_ACTOR', 23 | ) 24 | repository = environ.var( 25 | None, name='GITHUB_REPOSITORY', 26 | ) 27 | event_name = environ.var( 28 | None, name='GITHUB_EVENT_NAME', 29 | ) 30 | event_path = environ.var( 31 | None, converter=lambda p: p if p is None else Path(p), 32 | name='GITHUB_EVENT_PATH', 33 | ) 34 | workspace = environ.var( 35 | None, name='GITHUB_WORKSPACE', 36 | ) 37 | sha = environ.var( 38 | None, name='GITHUB_SHA', 39 | ) 40 | ref = environ.var( 41 | None, name='GITHUB_REF', 42 | ) 43 | token = environ.var( 44 | None, name='GITHUB_TOKEN', 45 | converter=lambda t: t if t is None else SecretStr(t), 46 | ) 47 | -------------------------------------------------------------------------------- /octomachinery/github/api/tokens.py: -------------------------------------------------------------------------------- 1 | """GitHub token types definitions.""" 2 | 3 | import attr 4 | 5 | # pylint: disable=relative-beyond-top-level 6 | from ..models.utils import SecretStr 7 | 8 | 9 | @attr.dataclass 10 | class GitHubToken: # pylint: disable=too-few-public-methods 11 | """Base class for GitHub tokens.""" 12 | 13 | _token_value: SecretStr = attr.ib( 14 | lambda s: SecretStr(s) if s is not None else s, 15 | ) 16 | 17 | def __str__(self): 18 | """Render the token as its string value.""" 19 | return str(self._token_value) 20 | 21 | 22 | @attr.dataclass 23 | # pylint: disable=too-few-public-methods 24 | class GitHubOAuthToken(GitHubToken): 25 | r"""GitHub OAuth Token. 26 | 27 | It can represent either App Installation token or a personal one. 28 | 29 | Ref: https://developer.github.com\ 30 | /apps/building-github-apps/authenticating-with-github-apps\ 31 | /#authenticating-as-an-installation 32 | """ 33 | 34 | 35 | @attr.dataclass 36 | # pylint: disable=too-few-public-methods 37 | class GitHubJWTToken(GitHubToken): 38 | r"""GitHub JSON Web Token. 39 | 40 | It represents GitHub App token. 41 | 42 | Ref: https://developer.github.com\ 43 | /apps/building-github-apps\ 44 | /authenticating-with-github-apps/#authenticating-as-a-github-app 45 | """ 46 | -------------------------------------------------------------------------------- /tests/app/routing/decorators_test.py: -------------------------------------------------------------------------------- 1 | """Test event routing decorator helpers.""" 2 | 3 | import pytest 4 | 5 | from octomachinery.app.routing.decorators import process_webhook_payload 6 | from octomachinery.github.models.events import GitHubEvent 7 | 8 | 9 | @process_webhook_payload 10 | def fake_event_handler(*, arg1, arg2): 11 | """Process fake test event.""" 12 | return arg1, arg2 13 | 14 | 15 | @pytest.mark.parametrize( 16 | 'incoming_event,is_successful', 17 | ( 18 | ({'arg1': 'y', 'arg2': 'x'}, True), 19 | ({'arg0': 'p', 'arg1': 'u', 'arg2': 'n'}, False), 20 | ({'arg1': 'z'}, False), 21 | ({'arg3': 's'}, False), 22 | ({}, False), 23 | ), 24 | ) 25 | def test_process_webhook_payload(incoming_event, is_successful): 26 | """Test that @process_webhook_payload unpacks event into kw-args.""" 27 | event = GitHubEvent( 28 | name=None, # type: ignore[arg-type] 29 | payload=incoming_event, 30 | ) 31 | 32 | if is_successful: 33 | assert ( 34 | # pylint: disable=missing-kwoa,too-many-function-args 35 | fake_event_handler(event) 36 | == tuple(incoming_event.values()) 37 | ) 38 | else: 39 | with pytest.raises(TypeError): 40 | # pylint: disable=missing-kwoa,too-many-function-args 41 | fake_event_handler(event) 42 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Georgia, serif, "apple color emoji", "segoe ui emoji", "android emoji", "emojisymbols", "emojione mozilla", "twemoji mozilla", "segoe ui symbol", "noto color emoji"; 3 | } 4 | 5 | div.sphinxsidebar h3, 6 | div.sphinxsidebar h4 { 7 | font-family: Georgia, serif, "apple color emoji", "segoe ui emoji", "android emoji", "emojisymbols", "emojione mozilla", "twemoji mozilla", "segoe ui symbol", "noto color emoji"; 8 | } 9 | 10 | div.sphinxsidebar input { 11 | font-family: Georgia, serif, "apple color emoji", "segoe ui emoji", "android emoji", "emojisymbols", "emojione mozilla", "twemoji mozilla", "segoe ui symbol", "noto color emoji"; 12 | } 13 | 14 | 15 | div.body h1, 16 | div.body h2, 17 | div.body h3, 18 | div.body h4, 19 | div.body h5, 20 | div.body h6 { 21 | font-family: Georgia, serif, "apple color emoji", "segoe ui emoji", "android emoji", "emojisymbols", "emojione mozilla", "twemoji mozilla", "segoe ui symbol", "noto color emoji"; 22 | } 23 | 24 | div.admonition p.admonition-title { 25 | font-family: Georgia, serif, "apple color emoji", "segoe ui emoji", "android emoji", "emojisymbols", "emojione mozilla", "twemoji mozilla", "segoe ui symbol", "noto color emoji"; 26 | } 27 | 28 | pre, tt, code { 29 | font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace, "apple color emoji", "segoe ui emoji", "android emoji", "emojisymbols", "emojione mozilla", "twemoji mozilla", "segoe ui symbol", "noto color emoji"; 30 | } 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # sphinxcontrib.apidoc autogenerated module references 107 | docs/reference/ 108 | -------------------------------------------------------------------------------- /octomachinery/runtime/utils.py: -------------------------------------------------------------------------------- 1 | """Runtime context helpers.""" 2 | 3 | from __future__ import annotations 4 | 5 | import typing 6 | from contextvars import ContextVar, Token 7 | 8 | 9 | class ContextLookupError(AttributeError): 10 | """Context var lookup error.""" 11 | 12 | 13 | class _ContextMap: 14 | __slots__ = '__map__', '__token_map__' 15 | 16 | def __init__(self, **initial_vars): 17 | self.__map__: typing.Dict[str, ContextVar[typing.Any]] = { 18 | k: ContextVar(v) for k, v in initial_vars.items() 19 | } 20 | """Storage for all context vars.""" 21 | 22 | self.__token_map__: typing.Dict[str, Token[typing.Any]] = {} 23 | """Storage for individual context var reset tokens.""" 24 | 25 | def __dir__(self): 26 | """Render a list of public attributes.""" 27 | return self.__map__.keys() 28 | 29 | def __getattr__(self, name): 30 | if name in ('__map__', '__token_map__'): 31 | return getattr(self, name) 32 | try: 33 | return self.__map__[name].get() 34 | except LookupError: 35 | raise ContextLookupError( 36 | f'No `{name}` present in the context', 37 | ) from None 38 | 39 | def __setattr__(self, name, value): 40 | if name in ('__map__', '__token_map__'): 41 | object.__setattr__(self, name, value) 42 | elif name in self.__map__: 43 | reset_token = self.__map__[name].set(value) 44 | self.__token_map__[name] = reset_token 45 | else: 46 | raise ContextLookupError(f'No `{name}` present in the context') 47 | 48 | def __delattr__(self, name): 49 | if name not in self.__map__: 50 | raise ContextLookupError(f'No `{name}` present in the context') 51 | reset_token = self.__token_map__[name] 52 | self.__map__[name].reset(reset_token) 53 | del self.__token_map__[name] 54 | -------------------------------------------------------------------------------- /octomachinery/github/entities/action.py: -------------------------------------------------------------------------------- 1 | """GitHub Action wrapper.""" 2 | 3 | from __future__ import annotations 4 | 5 | import logging 6 | import typing 7 | 8 | import attr 9 | 10 | # pylint: disable=relative-beyond-top-level 11 | from ..api.app_client import GitHubApp 12 | # pylint: disable=relative-beyond-top-level 13 | from ..api.raw_client import RawGitHubAPI 14 | # pylint: disable=relative-beyond-top-level 15 | from ..api.tokens import GitHubOAuthToken 16 | # pylint: disable=relative-beyond-top-level,import-error 17 | from ..models.events import GidgetHubActionEvent 18 | 19 | 20 | if typing.TYPE_CHECKING: 21 | # pylint: disable=relative-beyond-top-level 22 | from ...app.action.config import GitHubActionConfig 23 | 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | @attr.dataclass 29 | class GitHubAction(GitHubApp): 30 | """GitHub Action API wrapper.""" 31 | 32 | _metadata: GitHubActionConfig = attr.ib(default=None) 33 | """A GitHub Action metadata from envronment vars.""" 34 | 35 | @_metadata.validator 36 | def _verify_metadata_is_set(self, attribute, value): 37 | if value is None: 38 | raise ValueError(f'{attribute} must be set.') 39 | 40 | @property 41 | def event(self): # noqa: D401 42 | """Parsed GitHub Action event data.""" 43 | return GidgetHubActionEvent.from_file( 44 | self._metadata.event_name, # pylint: disable=no-member 45 | self._metadata.event_path, # pylint: disable=no-member 46 | ) 47 | 48 | @property 49 | def token(self): 50 | """Return GitHub Action access token.""" 51 | return GitHubOAuthToken( 52 | self._metadata.token, # pylint: disable=no-member 53 | ) 54 | 55 | @property 56 | def api_client(self): # noqa: D401 57 | """The GitHub App client.""" 58 | return RawGitHubAPI( 59 | token=self.token, 60 | session=self._http_session, 61 | user_agent=self._config.user_agent, 62 | ) 63 | -------------------------------------------------------------------------------- /octomachinery/github/models/utils.py: -------------------------------------------------------------------------------- 1 | """A collection of utility functions helping with models.""" 2 | 3 | import sys 4 | from datetime import datetime, timezone 5 | from functools import singledispatch 6 | 7 | 8 | @singledispatch 9 | def convert_datetime(datetime_obj) -> datetime: 10 | """Convert arbitrary object into a datetime instance.""" 11 | raise ValueError( 12 | f'The input arg type {type(datetime_obj)} is not supported', 13 | ) 14 | 15 | 16 | @convert_datetime.register 17 | def _convert_datetime_from_unixtime(date_unixtime: int) -> datetime: 18 | return datetime.fromtimestamp(date_unixtime, timezone.utc) 19 | 20 | 21 | @convert_datetime.register 22 | def _convert_datetime_from_string(date_string: str) -> datetime: 23 | if not date_string: 24 | raise ValueError( 25 | f'The input arg {date_string!r} is unsupported', 26 | ) 27 | date_string = date_string.replace('.000Z', '.000000Z') 28 | if '.' not in date_string: 29 | date_string = date_string.replace('Z', '.000000Z') 30 | if '+' not in date_string: 31 | date_string += '+00:00' 32 | 33 | # datetime.fromisoformat() doesn't understand microseconds 34 | return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ%z') 35 | 36 | 37 | class SecretStr(str): 38 | """String that censors its __repr__ if called from another repr.""" 39 | 40 | def __repr__(self): 41 | """Produce a string representation.""" 42 | frame_depth = 1 43 | 44 | try: 45 | while True: 46 | frame = sys._getframe( # pylint: disable=protected-access 47 | frame_depth, 48 | ) 49 | frame_depth += 1 50 | 51 | if frame.f_code.co_name == '__repr__': 52 | return '' 53 | except ValueError: 54 | pass 55 | 56 | return super().__repr__() 57 | 58 | 59 | class SuperSecretStr(SecretStr): 60 | """String that always censors its __repr__.""" 61 | 62 | def __repr__(self): 63 | """Produce a string representation.""" 64 | return '' 65 | -------------------------------------------------------------------------------- /octomachinery/github/models/action_outcomes.py: -------------------------------------------------------------------------------- 1 | """Processing outcomes for use from within GitHub Action env.""" 2 | 3 | import logging 4 | 5 | import attr 6 | 7 | 8 | __all__ = ('ActionSuccess', 'ActionNeutral', 'ActionFailure') 9 | 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | @attr.dataclass 15 | class ActionOutcome: # pylint: disable=too-few-public-methods 16 | """GitHub Action processing outcome.""" 17 | 18 | message: str 19 | return_code: int 20 | 21 | def raise_it(self): 22 | """Print the message and exit the program with current code.""" 23 | logger.info( 24 | 'Terminating the GitHub Action processing: %s', 25 | self.message, 26 | ) 27 | raise SystemExit(self.return_code) 28 | 29 | 30 | @attr.dataclass 31 | # pylint: disable=too-few-public-methods 32 | class ActionSuccess(ActionOutcome): 33 | """GitHub Action successful outcome.""" 34 | 35 | return_code: int = attr.ib(default=0, init=False) 36 | 37 | 38 | @attr.dataclass 39 | # pylint: disable=too-few-public-methods 40 | class ActionFailure(ActionOutcome): 41 | """GitHub Action failed outcome.""" 42 | 43 | return_code: int = attr.ib(default=1) 44 | 45 | @return_code.validator 46 | def _validate_return_code( 47 | self, 48 | attribute, # pylint: disable=unused-argument 49 | value, 50 | ): 51 | if value in NON_FAIL_MODELS: 52 | raise ValueError( 53 | f'Return code of `{value}` is illegal to use for failure ' 54 | f'outcome. Use {NON_FAIL_MODELS[value]} instead', 55 | ) 56 | 57 | 58 | @attr.dataclass 59 | # pylint: disable=too-few-public-methods 60 | class ActionNeutral(ActionOutcome): 61 | """GitHub Action neutral outcome.""" 62 | 63 | # NOTE: It's EX_CONFIG under BSD and EREMCHG under GNU/Linux 64 | # NOTE: that's why we are using just 78 conventional constant 65 | # NOTE: here... 66 | # NOTE: Ref: 67 | # https://developer.github.com/actions/creating-github-actions\ 68 | # /accessing-the-runtime-environment/#exit-codes-and-statuses 69 | return_code: int = attr.ib(default=78, init=False) 70 | 71 | 72 | NON_FAIL_MODELS = { 73 | 0: 'ActionSuccess', 74 | 78: 'ActionNeutral', 75 | } 76 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = 3 | # `pytest-xdist` == -n auto: 4 | --numprocesses=auto 5 | 6 | # Show 10 slowest invocations: 7 | --durations=10 8 | 9 | # A bit of verbosity doesn't hurt: 10 | -v 11 | 12 | # Report all the things == -rxXs: 13 | -ra 14 | 15 | # Show values of the local vars in errors: 16 | --showlocals 17 | 18 | # Autocollect and invoke the doctests from all modules: 19 | # https://docs.pytest.org/en/stable/doctest.html 20 | --doctest-modules 21 | 22 | # Dump the test results in junit format: 23 | --junitxml=.tox/tmp/test-results/pytest/results.xml 24 | 25 | # Fail on config parsing warnings: 26 | # --strict-config 27 | 28 | # Fail on non-existing markers: 29 | # * Deprecated since v6.2.0 but may be reintroduced later covering a 30 | # broader scope: 31 | # --strict 32 | # * Exists since v4.5.0 (advised to be used instead of `--strict`): 33 | --strict-markers 34 | 35 | # `pytest-cov`: 36 | # `pytest-cov`, "-p" preloads the module early: 37 | -p pytest_cov 38 | --no-cov-on-fail 39 | --cov=octomachinery 40 | --cov=tests/ 41 | --cov-branch 42 | --cov-report=term-missing:skip-covered 43 | --cov-report=html:.tox/tmp/test-results/pytest/cov/ 44 | --cov-report=xml 45 | --cov-context=test 46 | --cov-config=.coveragerc 47 | 48 | doctest_optionflags = ALLOW_UNICODE ELLIPSIS 49 | 50 | # Marks tests with an empty parameterset as xfail(run=False) 51 | empty_parameter_set_mark = xfail 52 | 53 | faulthandler_timeout = 30 54 | 55 | filterwarnings = 56 | error 57 | # drop this once aiohttp>4 is out 58 | ignore:The loop argument is deprecated since Python 3.8, and scheduled for removal in Python 3.10.:DeprecationWarning 59 | 60 | junit_duration_report = call 61 | # xunit1 contains more metadata than xunit2 so it's better for CI UIs: 62 | junit_family = xunit1 63 | junit_logging = all 64 | junit_log_passing_tests = true 65 | junit_suite_name = octomachinery_test_suite 66 | 67 | # A mapping of markers to their descriptions allowed in strict mode: 68 | markers = 69 | 70 | minversion = 5.3.3 71 | 72 | # Optimize pytest's lookup by restricting potentially deep dir tree scan: 73 | norecursedirs = 74 | build 75 | dist 76 | docs 77 | octomachinery.egg-info 78 | .cache 79 | .eggs 80 | .git 81 | .github 82 | .tox 83 | 84 | testpaths = tests/ 85 | 86 | xfail_strict = true 87 | -------------------------------------------------------------------------------- /octomachinery/app/server/runner.py: -------------------------------------------------------------------------------- 1 | """Octomachinery CLI runner.""" 2 | 3 | import logging 4 | import sys 5 | from typing import Iterable, Optional 6 | 7 | from aiohttp.web_runner import GracefulExit 8 | from anyio import run as run_until_complete 9 | 10 | import attr 11 | 12 | # pylint: disable=relative-beyond-top-level 13 | from ..config import BotAppConfig 14 | # pylint: disable=relative-beyond-top-level 15 | from ..routing import WEBHOOK_EVENTS_ROUTER 16 | # pylint: disable=relative-beyond-top-level 17 | from ..routing.abc import OctomachineryRouterBase 18 | # pylint: disable=relative-beyond-top-level 19 | from .config import WebServerConfig 20 | # pylint: disable=relative-beyond-top-level 21 | from .machinery import run_forever as run_server_forever 22 | 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | def run( 28 | *, 29 | name: Optional[str] = None, 30 | version: Optional[str] = None, 31 | url: Optional[str] = None, 32 | config: Optional[BotAppConfig] = None, 33 | event_routers: Optional[Iterable[OctomachineryRouterBase]] = None, 34 | ): 35 | """Start up a server using CLI args for host and port.""" 36 | if event_routers is None: 37 | event_routers = {WEBHOOK_EVENTS_ROUTER} 38 | 39 | if ( 40 | config is not None and 41 | (name is not None or version is not None or url is not None) 42 | ): 43 | raise TypeError( 44 | 'run() takes either a BotAppConfig instance as a config argument ' 45 | 'or name, version and url arguments.', 46 | ) 47 | if config is None: 48 | config = BotAppConfig.from_dotenv( 49 | app_name=name, 50 | app_version=version, 51 | app_url=url, 52 | ) 53 | if len(sys.argv) > 2: 54 | config = attr.evolve( # type: ignore[misc] 55 | config, 56 | server=WebServerConfig(*sys.argv[1:3]), 57 | ) 58 | 59 | logging.basicConfig( 60 | level=logging.DEBUG 61 | if config.runtime.debug # pylint: disable=no-member 62 | else logging.INFO, 63 | ) 64 | if config.runtime.debug: # pylint: disable=no-member 65 | logger.debug( 66 | ' App version: %s '.center(50, '='), 67 | config.github.app_version, 68 | ) 69 | 70 | try: 71 | run_until_complete(run_server_forever, config, event_routers) 72 | except (GracefulExit, KeyboardInterrupt): 73 | logger.info(' Exiting the app '.center(50, '=')) 74 | -------------------------------------------------------------------------------- /octomachinery/app/config.py: -------------------------------------------------------------------------------- 1 | """GitHub App/bot configuration.""" 2 | 3 | import os 4 | from functools import lru_cache 5 | from typing import Optional 6 | 7 | import environ 8 | import envparse 9 | 10 | # pylint: disable=relative-beyond-top-level 11 | from ..github.config.app import GitHubAppIntegrationConfig 12 | # pylint: disable=relative-beyond-top-level 13 | from .action.config import GitHubActionConfig 14 | # pylint: disable=relative-beyond-top-level 15 | from .runtime.config import RuntimeConfig 16 | # pylint: disable=relative-beyond-top-level 17 | from .server.config import WebServerConfig 18 | 19 | 20 | @environ.config 21 | class BotAppConfig: 22 | """Bot app config. 23 | 24 | Construct it as follows:: 25 | >>> from octomachinery.app.config import BotAppConfig 26 | >>> config = BotAppConfig.from_dotenv() # for dev env 27 | >>> config = BotAppConfig.from_env() # for pure env 28 | >>> 29 | """ 30 | 31 | github = environ.group(GitHubAppIntegrationConfig) 32 | action = environ.group(GitHubActionConfig) 33 | server = environ.group(WebServerConfig) 34 | runtime = environ.group(RuntimeConfig) 35 | 36 | @classmethod 37 | @lru_cache(maxsize=1) 38 | def from_dotenv( 39 | cls, 40 | *, 41 | app_name: Optional[str] = None, 42 | app_version: Optional[str] = None, 43 | app_url: Optional[str] = None, 44 | ): 45 | """Return an initialized dev config instance. 46 | 47 | Read .env into env vars before that. 48 | """ 49 | envparse.Env.read_envfile( 50 | '.env', # Making it relative to CWD, relative to caller if None 51 | ) 52 | return cls.from_env( 53 | app_name=app_name, 54 | app_version=app_version, 55 | app_url=app_url, 56 | ) 57 | 58 | @classmethod 59 | @lru_cache(maxsize=1) 60 | def from_env( 61 | cls, 62 | *, 63 | app_name: Optional[str] = None, 64 | app_version: Optional[str] = None, 65 | app_url: Optional[str] = None, 66 | ): 67 | """Return an initialized config instance.""" 68 | env_vars = dict(os.environ) 69 | if app_name is not None: 70 | env_vars['OCTOMACHINERY_APP_NAME'] = app_name 71 | if app_version is not None: 72 | env_vars['OCTOMACHINERY_APP_VERSION'] = app_version 73 | if app_url is not None: 74 | env_vars['OCTOMACHINERY_APP_URL'] = app_url 75 | return environ.to_config(cls, environ=env_vars) 76 | -------------------------------------------------------------------------------- /octomachinery/github/entities/app_installation.py: -------------------------------------------------------------------------------- 1 | """GitHub App Installation wrapper.""" 2 | 3 | from __future__ import annotations 4 | 5 | import logging 6 | import typing 7 | 8 | import attr 9 | 10 | # pylint: disable=relative-beyond-top-level 11 | from ..api.raw_client import RawGitHubAPI 12 | # pylint: disable=relative-beyond-top-level 13 | from ..api.tokens import GitHubOAuthToken 14 | # pylint: disable=relative-beyond-top-level 15 | from ..models import GitHubAppInstallation as GitHubAppInstallationModel 16 | from ..models import GitHubInstallationAccessToken 17 | 18 | 19 | if typing.TYPE_CHECKING: 20 | from ..api.app_client import GitHubApp 21 | 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | @attr.dataclass 27 | class GitHubAppInstallation: 28 | """GitHub App Installation API wrapper.""" 29 | 30 | _metadata: GitHubAppInstallationModel 31 | """A GitHub Installation metadata from GitHub webhook.""" 32 | _github_app: GitHubApp 33 | """A GitHub App the Installation is associated with.""" 34 | _token: GitHubInstallationAccessToken = attr.ib(init=False, default=None) 35 | """A GitHub Installation token for GitHub API.""" 36 | 37 | @property 38 | def app(self): 39 | """Bound GitHub App instance.""" 40 | return self._github_app 41 | 42 | async def get_token(self): 43 | """Retrieve installation access token from GitHub API.""" 44 | return GitHubInstallationAccessToken( 45 | **( 46 | await self.app.api_client.post( 47 | self._metadata.access_tokens_url, 48 | data=b'', 49 | preview_api_version='machine-man', 50 | ) 51 | ), 52 | ) 53 | 54 | async def _refresh_api_token(self): 55 | """Extract installation access token value. 56 | 57 | Refreshes it as needed. 58 | """ 59 | if self._token is None or self._token.expired: 60 | self._token = await self.get_token() 61 | 62 | return GitHubOAuthToken(self._token.token) 63 | 64 | @property 65 | def api_client(self): # noqa: D401 66 | """The GitHub App Installation client.""" 67 | return RawGitHubAPI( 68 | # pylint: disable=fixme 69 | token=self._refresh_api_token, # type: ignore[arg-type] # FIXME 70 | # pylint: disable=protected-access 71 | session=self.app._http_session, 72 | # pylint: disable=protected-access 73 | user_agent=self.app._config.user_agent, 74 | ) 75 | -------------------------------------------------------------------------------- /octomachinery/github/api/utils.py: -------------------------------------------------------------------------------- 1 | """Utilitary helpers.""" 2 | 3 | from functools import wraps 4 | from inspect import Parameter, Signature 5 | from types import AsyncGeneratorType 6 | from typing import Any, Dict, Tuple 7 | 8 | from gidgethub.sansio import accept_format 9 | 10 | 11 | def mark_uninitialized_in_repr(cls): 12 | """Patch __repr__ for uninitialized instances.""" 13 | orig_repr = cls.__repr__ 14 | 15 | @wraps(orig_repr) 16 | def new_repr(self): 17 | if not self.is_initialized: 18 | return f'{self.__class__.__name__}()' 19 | return orig_repr(self) 20 | cls.__repr__ = new_repr 21 | return cls 22 | 23 | 24 | def accept_preview_version(wrapped_coroutine): 25 | """Extend keyword-args with `preview_api_version`.""" 26 | @wraps(wrapped_coroutine) 27 | def coroutine_wrapper( 28 | *args: Tuple[Any], **kwargs: Dict[str, Any], 29 | ) -> AsyncGeneratorType: # type: ignore[type-arg] 30 | accept_media = kwargs.pop('accept', None) 31 | preview_api_version = kwargs.pop('preview_api_version', None) 32 | 33 | if preview_api_version is not None: 34 | accept_media = accept_format( 35 | version=f'{preview_api_version}-preview', 36 | ) 37 | if accept_media is not None: 38 | kwargs['accept'] = accept_media 39 | 40 | coroutine_instance = wrapped_coroutine(*args, **kwargs) 41 | is_async_generator = isinstance(coroutine_instance, AsyncGeneratorType) 42 | 43 | if not is_async_generator: 44 | async def async_function_wrapper(): 45 | return await coroutine_instance 46 | return async_function_wrapper() 47 | 48 | async def async_generator_wrapper(): 49 | async for result_item in coroutine_instance: 50 | yield result_item 51 | 52 | return async_generator_wrapper() 53 | 54 | original_wrapped_signature = Signature.from_callable(wrapped_coroutine) 55 | original_callable_params = original_wrapped_signature.parameters 56 | wrapped_callable_params = list(original_callable_params.values()) 57 | 58 | accept_pos = list(original_callable_params.keys()).index('accept') 59 | preview_param = Parameter( 60 | name='preview_api_version', 61 | annotation='Optional[str]', 62 | default=None, 63 | kind=Parameter.KEYWORD_ONLY, 64 | ) 65 | wrapped_callable_params.insert(accept_pos, preview_param) 66 | 67 | coroutine_wrapper.__signature__ = ( # type: ignore[attr-defined] 68 | original_wrapped_signature.replace(parameters=wrapped_callable_params) 69 | ) 70 | 71 | return coroutine_wrapper 72 | -------------------------------------------------------------------------------- /tests/utils/asynctools_test.py: -------------------------------------------------------------------------------- 1 | """Test for asynchronous operations utility functions.""" 2 | 3 | import pytest 4 | 5 | from octomachinery.utils.asynctools import amap, dict_to_kwargs_cb, try_await 6 | 7 | 8 | def sync_power2(val): 9 | """Raise x to the power of 2.""" 10 | return val ** 2 11 | 12 | 13 | @pytest.mark.anyio 14 | async def async_power2(val): 15 | """Raise x to the power of 2 asynchronously.""" 16 | return sync_power2(val) 17 | 18 | 19 | @pytest.mark.parametrize( 20 | 'callback_func', 21 | ( 22 | sync_power2, 23 | async_power2, 24 | ), 25 | ) 26 | @pytest.mark.anyio 27 | @pytest.mark.usefixtures('event_loop') 28 | async def test_amap(callback_func): 29 | """Test that async map works for both sync and async callables.""" 30 | async def async_iter(*args, **kwargs): 31 | for _ in range(*args, **kwargs): 32 | yield _ 33 | 34 | test_range = 5 35 | 36 | actual_result = [ 37 | i async for i in amap(callback_func, async_iter(test_range)) 38 | ] 39 | expected_result = [ 40 | await try_await(callback_func(i)) for i in range(test_range) 41 | ] 42 | assert actual_result == expected_result 43 | 44 | 45 | @pytest.mark.parametrize( 46 | 'callback_func', 47 | ( 48 | sync_power2, 49 | async_power2, 50 | ), 51 | ) 52 | @pytest.mark.anyio 53 | @pytest.mark.usefixtures('event_loop') 54 | async def test_dict_to_kwargs_cb(callback_func): 55 | """Test that input dict is turned into given (a)sync callable args.""" 56 | test_val = 5 57 | test_dict = {'val': test_val} 58 | 59 | actual_result = await dict_to_kwargs_cb(callback_func)(test_dict) 60 | expected_result = await try_await(callback_func(test_val)) 61 | assert actual_result == expected_result 62 | 63 | 64 | @pytest.mark.parametrize( 65 | 'callback_func,callback_arg', 66 | ( 67 | (sync_power2, 3), 68 | (async_power2, 8), 69 | ), 70 | ) 71 | @pytest.mark.anyio 72 | @pytest.mark.usefixtures('event_loop') 73 | async def test_try_await(callback_func, callback_arg): 74 | """Test that result is awaited regardless of (a)sync func type.""" 75 | actual_result = await try_await(callback_func(callback_arg)) 76 | expected_result = callback_arg ** 2 77 | assert actual_result == expected_result 78 | 79 | 80 | @pytest.mark.anyio 81 | @pytest.mark.usefixtures('event_loop') 82 | async def test_try_await_bypass_errors(): 83 | """Test that internal callback exceptions are propagated.""" 84 | async def break_callback(): 85 | raise TypeError('It is broken') 86 | 87 | with pytest.raises(TypeError, match='It is broken'): 88 | await try_await(break_callback()) 89 | -------------------------------------------------------------------------------- /octomachinery/github/config/app.py: -------------------------------------------------------------------------------- 1 | """Config schema for a GitHub App instance details.""" 2 | import environ 3 | 4 | # pylint: disable=relative-beyond-top-level 5 | from ..models.private_key import GitHubPrivateKey 6 | # pylint: disable=relative-beyond-top-level 7 | from ..models.utils import SecretStr 8 | 9 | 10 | def validate_is_not_none_if_app( 11 | self, # pylint: disable=unused-argument 12 | attr, value, 13 | ): 14 | """Forbid None value in a GitHub App context.""" 15 | # pylint: disable=relative-beyond-top-level,import-outside-toplevel 16 | from ...app.runtime.utils import detect_env_mode 17 | 18 | if value is None and detect_env_mode() == 'app': 19 | raise ValueError( 20 | f'GitHub App must provide a proper value for {attr!r}', 21 | ) 22 | 23 | 24 | def validate_fingerprint_if_present(instance, _attribute, value): 25 | r"""Validate that the private key matches the fingerprint pin. 26 | 27 | :raises ValueError: if the fingerprint pin is present \ 28 | but doesn't match the private key 29 | """ 30 | if not value: 31 | return 32 | 33 | if instance.private_key.matches_fingerprint(value): 34 | return 35 | 36 | raise ValueError( 37 | 'The private key provided (with a fingerprint of ' 38 | f'{instance.private_key.fingerprint!s}) does not match ' 39 | f'the pinned fingerprint value of {value!s}', 40 | ) 41 | 42 | 43 | @environ.config 44 | class GitHubAppIntegrationConfig: # pylint: disable=too-few-public-methods 45 | """GitHub App auth related config.""" 46 | 47 | app_id = environ.var( 48 | None, 49 | name='GITHUB_APP_IDENTIFIER', 50 | validator=validate_is_not_none_if_app, 51 | ) 52 | private_key = environ.var( 53 | None, 54 | name='GITHUB_PRIVATE_KEY', 55 | converter=lambda raw_data: 56 | None if raw_data is None else GitHubPrivateKey(raw_data.encode()), 57 | validator=validate_is_not_none_if_app, 58 | ) 59 | private_key_fingerprint = environ.var( 60 | None, 61 | name='GITHUB_PRIVATE_KEY_FINGERPRINT', 62 | validator=validate_fingerprint_if_present, 63 | ) 64 | webhook_secret = environ.var( 65 | None, name='GITHUB_WEBHOOK_SECRET', 66 | converter=lambda s: SecretStr(s) if s is not None else s, 67 | ) 68 | 69 | app_name = environ.var(None, name='OCTOMACHINERY_APP_NAME') 70 | app_version = environ.var(None, name='OCTOMACHINERY_APP_VERSION') 71 | app_url = environ.var(None, name='OCTOMACHINERY_APP_URL') 72 | 73 | @property 74 | def user_agent(self): # noqa: D401 75 | """The User-Agent value to use when hitting GitHub API.""" 76 | return f'{self.app_name}/{self.app_version} (+{self.app_url})' 77 | -------------------------------------------------------------------------------- /octomachinery/app/action/runner.py: -------------------------------------------------------------------------------- 1 | """Octomachinery CLI runner for GitHub Action environments.""" 2 | 3 | import asyncio 4 | import logging 5 | from typing import Iterable, Optional 6 | 7 | from aiohttp.client import ClientSession 8 | 9 | # pylint: disable=relative-beyond-top-level 10 | from ...github.entities.action import GitHubAction 11 | # pylint: disable=relative-beyond-top-level 12 | from ...github.errors import GitHubActionError 13 | # pylint: disable=relative-beyond-top-level 14 | from ...github.models.action_outcomes import ( 15 | ActionFailure, ActionNeutral, ActionSuccess, 16 | ) 17 | # pylint: disable=relative-beyond-top-level 18 | from ..config import BotAppConfig 19 | # pylint: disable=relative-beyond-top-level 20 | from ..routing import WEBHOOK_EVENTS_ROUTER 21 | # pylint: disable=relative-beyond-top-level 22 | from ..routing.abc import OctomachineryRouterBase 23 | # pylint: disable=relative-beyond-top-level 24 | from ..routing.webhooks_dispatcher import route_github_event 25 | 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | 30 | async def process_github_action(config, event_routers): 31 | """Schedule GitHub Action event for processing.""" 32 | logger.info('Processing GitHub Action event...') 33 | 34 | async with ClientSession() as http_client_session: 35 | github_action = GitHubAction( 36 | metadata=config.action, 37 | http_session=http_client_session, 38 | config=config.github, 39 | event_routers=event_routers, 40 | ) 41 | logger.info('GitHub Action=%r', config.action) 42 | 43 | await route_github_event( 44 | github_event=github_action.event, 45 | github_app=github_action, 46 | ) 47 | return ActionSuccess('GitHub Action has been processed') 48 | 49 | 50 | def run( 51 | *, 52 | config: Optional[BotAppConfig] = None, 53 | event_routers: Optional[Iterable[OctomachineryRouterBase]] = None, 54 | ) -> None: 55 | """Start up a server using CLI args for host and port.""" 56 | if event_routers is None: 57 | event_routers = {WEBHOOK_EVENTS_ROUTER} 58 | 59 | if config is None: 60 | config = BotAppConfig.from_dotenv() 61 | 62 | logging.basicConfig( 63 | level=logging.DEBUG 64 | if config.runtime.debug # pylint: disable=no-member 65 | else logging.INFO, 66 | ) 67 | 68 | try: 69 | processing_outcome = asyncio.run( 70 | process_github_action(config, event_routers), 71 | ) 72 | except GitHubActionError as action_error: 73 | action_error.terminate_action() 74 | except KeyboardInterrupt: 75 | ActionNeutral('Action processing interrupted by user').raise_it() 76 | except Exception: # pylint: disable=broad-except 77 | err_msg = 'Action processing failed unexpectedly' 78 | logger.exception(err_msg) 79 | ActionFailure(err_msg).raise_it() 80 | else: 81 | processing_outcome.raise_it() 82 | -------------------------------------------------------------------------------- /octomachinery/app/runtime/installation_utils.py: -------------------------------------------------------------------------------- 1 | """Utility helpers for App/Action installations.""" 2 | 3 | import typing 4 | from base64 import b64decode 5 | from http import HTTPStatus 6 | from io import StringIO 7 | from pathlib import Path 8 | 9 | import gidgethub 10 | 11 | import yaml 12 | 13 | # pylint: disable=relative-beyond-top-level 14 | from ...runtime.context import RUNTIME_CONTEXT 15 | 16 | 17 | def _get_file_contents_from_fs(file_name: str) -> typing.Optional[str]: 18 | """Read file contents from file system checkout. 19 | 20 | This code path is synchronous. 21 | 22 | It doesn't matter much in GitHub Actions 23 | but can be refactored later. 24 | """ 25 | config_path = Path('.') / file_name 26 | 27 | try: 28 | return config_path.read_text() 29 | except FileNotFoundError: 30 | return None 31 | 32 | 33 | async def _get_file_contents_from_api( 34 | file_name: str, 35 | ref: typing.Optional[str], 36 | ) -> typing.Optional[str]: 37 | """Read file contents using GitHub API.""" 38 | github_api = RUNTIME_CONTEXT.app_installation_client 39 | repo_slug = RUNTIME_CONTEXT.github_event.payload['repository']['full_name'] 40 | 41 | api_query_params = f'?ref={ref}' if ref else '' 42 | try: 43 | config_response = await github_api.getitem( 44 | f'/repos/{repo_slug}/contents' 45 | f'/{file_name}{api_query_params}', 46 | ) 47 | except gidgethub.BadRequest as http_bad_req: 48 | if http_bad_req.status_code == HTTPStatus.NOT_FOUND: 49 | return None 50 | 51 | raise 52 | 53 | config_file_found = ( 54 | config_response.get('encoding') == 'base64' and 55 | 'content' in config_response 56 | ) 57 | if not config_file_found: 58 | return None 59 | 60 | return b64decode(config_response['content']).decode() 61 | 62 | 63 | async def read_file_contents_from_repo( 64 | *, 65 | file_path: str, 66 | ref: typing.Optional[str] = None, 67 | ) -> typing.Optional[str]: 68 | """Get a config object from the current installation. 69 | 70 | Read from file system checkout in case of GitHub Action env. 71 | Grab it via GitHub API otherwise. 72 | 73 | Usage:: 74 | 75 | >>> from octomachinery.app.runtime.installation_utils import ( 76 | ... read_file_contents_from_repo 77 | ... ) 78 | >>> await read_file_contents_from_repo( 79 | ... '/file/path.txt', 80 | ... ref='bdeaf38', 81 | ... ) 82 | """ 83 | if RUNTIME_CONTEXT.IS_GITHUB_ACTION and ref is None: 84 | return _get_file_contents_from_fs(file_path) 85 | 86 | return await _get_file_contents_from_api(file_path, ref) 87 | 88 | 89 | async def get_installation_config( 90 | *, 91 | config_name: str = 'config.yml', 92 | ref: typing.Optional[str] = None, 93 | ) -> typing.Mapping[str, typing.Any]: 94 | """Get a config object from the current installation. 95 | 96 | Read from file system checkout in case of GitHub Action env. 97 | Grab it via GitHub API otherwise. 98 | 99 | Usage:: 100 | 101 | >>> from octomachinery.app.runtime.installation_utils import ( 102 | ... get_installation_config 103 | ... ) 104 | >>> await get_installation_config() 105 | """ 106 | config_path = f'.github/{config_name}' 107 | 108 | config_content = await read_file_contents_from_repo( 109 | file_path=config_path, 110 | ref=ref, 111 | ) 112 | 113 | if config_content is None: 114 | return {} 115 | 116 | return yaml.load(StringIO(config_content), Loader=yaml.SafeLoader) 117 | -------------------------------------------------------------------------------- /tests/app/action/runner_test.py: -------------------------------------------------------------------------------- 1 | """Actions processor test suite.""" 2 | 3 | import json 4 | 5 | import pytest 6 | 7 | from octomachinery.app.action.config import GitHubActionConfig 8 | from octomachinery.app.action.runner import run 9 | from octomachinery.app.config import BotAppConfig 10 | from octomachinery.app.routing import process_event, process_event_actions 11 | from octomachinery.app.runtime.config import RuntimeConfig 12 | from octomachinery.app.server.config import WebServerConfig 13 | from octomachinery.github.config.app import GitHubAppIntegrationConfig 14 | from octomachinery.github.errors import GitHubActionError 15 | from octomachinery.github.models.action_outcomes import ActionNeutral 16 | 17 | 18 | @process_event('unmatched_event', action='happened') 19 | async def unmatched_event_happened(action): # pylint: disable=unused-argument 20 | """Handle an unmatched event.""" 21 | 22 | 23 | @process_event('check_run', action='created') 24 | async def check_run_created(action): # pylint: disable=unused-argument 25 | """Handle a check_run event.""" 26 | raise RuntimeError('Emulate an unhandled error in the handler') 27 | 28 | 29 | @process_event_actions('neutral_event', {'qwerty'}) 30 | async def neutral_event_qwerty(action): # pylint: disable=unused-argument 31 | """Handle a neutral_event.""" 32 | raise GitHubActionError(ActionNeutral('Neutral outcome')) 33 | 34 | 35 | @process_event_actions('neutral_event') 36 | async def neutral_event_dummy(action): # pylint: disable=unused-argument 37 | """Handle any neutral event.""" 38 | 39 | 40 | @pytest.fixture 41 | def event_file(tmp_path_factory, request): 42 | """Generate a sample JSON event file.""" 43 | gh_event = ( 44 | tmp_path_factory.mktemp('github-action-event') / 45 | 'github-workflow-event.json' 46 | ) 47 | with gh_event.open('w') as gh_event_file: 48 | json.dump( 49 | { 50 | 'action': request.param, # 'created', 51 | }, 52 | gh_event_file, 53 | ) 54 | return gh_event 55 | 56 | 57 | @pytest.fixture 58 | def config(monkeypatch, event_file, request): 59 | """Create a dummy GitHub Action config.""" 60 | monkeypatch.setattr( 61 | 'octomachinery.app.runtime.utils.detect_env_mode', 62 | lambda: 'action', 63 | ) 64 | # pylint: disable=fixme 65 | return BotAppConfig( # type: ignore[call-arg] # FIXME 66 | github=GitHubAppIntegrationConfig(), 67 | # pylint: disable=fixme 68 | action=GitHubActionConfig( # type: ignore[call-arg] # FIXME 69 | workflow='Test Workflow', 70 | action='Test Action', 71 | actor='username-or-bot', 72 | repository='org/repo', 73 | event_name=request.param, # 'check_run' 74 | event_path=str(event_file), # '/github/workflow/event.json' 75 | workspace='/github/workspace', 76 | sha='e6d4abcb8a6cd989d41ee', 77 | ref='refs/heads/master', 78 | token='8sdfn12lifds8sdvh832n32f9jew', 79 | ), 80 | server=WebServerConfig(), 81 | runtime=RuntimeConfig(), 82 | ) 83 | 84 | 85 | @pytest.mark.parametrize( 86 | 'config, event_file, expected_return_code', 87 | ( 88 | ('check_run', 'created', 1), 89 | ('unmatched_event', 'closed', 0), 90 | ('neutral_event', 'qwerty', 78), 91 | ), 92 | indirect=('config', 'event_file'), 93 | ) 94 | def test_action_processing_return_code(config, expected_return_code): 95 | """Test an empty action processing run.""" 96 | with pytest.raises( 97 | SystemExit, 98 | match=r'^' 99 | f'{expected_return_code}' 100 | r'$', 101 | ): 102 | run(config=config) 103 | -------------------------------------------------------------------------------- /octomachinery/routing/routers.py: -------------------------------------------------------------------------------- 1 | """Octomachinery event dispatchers collection.""" 2 | 3 | import asyncio 4 | from contextlib import suppress 5 | from typing import Any, Iterator, Set, Union 6 | 7 | from gidgethub.routing import AsyncCallback 8 | from gidgethub.routing import Router as _GidgetHubRouter 9 | 10 | from ..github.models.events import ( 11 | GidgetHubWebhookEvent, GitHubEvent, _GidgetHubEvent, 12 | ) 13 | from ..utils.asynctools import aio_gather 14 | from .abc import OctomachineryRouterBase 15 | 16 | 17 | __all__ = ( 18 | 'GidgetHubRouterBase', 19 | 'ConcurrentRouter', 20 | 'NonBlockingConcurrentRouter', 21 | ) 22 | 23 | 24 | class GidgetHubRouterBase(_GidgetHubRouter, OctomachineryRouterBase): 25 | """GidgetHub-based router exposing callback matching separately.""" 26 | 27 | def emit_routes_for( 28 | self, event_name: str, event_payload: Any, 29 | ) -> Iterator[AsyncCallback]: 30 | """Emit callbacks that match given event and payload. 31 | 32 | :param str event_name: name of the GitHub event 33 | :param str event_payload: details of the GitHub event 34 | 35 | :yields: coroutine event handlers 36 | """ 37 | with suppress(KeyError): 38 | yield from self._shallow_routes[event_name] 39 | 40 | try: 41 | deep_routes = self._deep_routes[event_name] 42 | except KeyError: 43 | return 44 | 45 | for payload_key, payload_values in deep_routes.items(): 46 | if payload_key not in event_payload: 47 | continue 48 | event_value = event_payload[payload_key] 49 | if event_value not in payload_values: 50 | continue 51 | yield from payload_values[event_value] 52 | 53 | async def dispatch( 54 | self, event: Union[GidgetHubWebhookEvent, _GidgetHubEvent], 55 | *args: Any, **kwargs: Any, 56 | ) -> None: 57 | """Invoke handler tasks for the given event sequentially.""" 58 | if isinstance(event, _GidgetHubEvent): 59 | event = GidgetHubWebhookEvent.from_gidgethub(event) 60 | 61 | callback_gen = self.emit_routes_for(event.name, event.payload) 62 | callback_coros = (cb(event, *args, **kwargs) for cb in callback_gen) 63 | 64 | for coro in callback_coros: 65 | await coro 66 | 67 | 68 | class ConcurrentRouter(GidgetHubRouterBase): 69 | """GitHub event router invoking event handlers simultaneously.""" 70 | 71 | async def dispatch( 72 | self, event: GitHubEvent, 73 | *args: Any, **kwargs: Any, 74 | ) -> None: 75 | """Invoke coroutine callbacks for the given event together.""" 76 | callback_gen = self.emit_routes_for(event.name, event.payload) 77 | callback_coros = (cb(event, *args, **kwargs) for cb in callback_gen) 78 | 79 | await aio_gather(*callback_coros) 80 | 81 | 82 | class NonBlockingConcurrentRouter(ConcurrentRouter): 83 | """Non-blocking GitHub event router scheduling handler tasks.""" 84 | 85 | def __init__(self, *args, **kwargs): 86 | """Initialize NonBlockingConcurrentRouter.""" 87 | super().__init__(*args, **kwargs) 88 | # NOTE: For some reason, mypy doesn't accept anything except Any here: 89 | self._event_handler_tasks: Set[Any] = set() 90 | 91 | async def dispatch( 92 | self, event: GitHubEvent, 93 | *args: Any, **kwargs: Any, 94 | ) -> None: 95 | """Schedule coroutine callbacks for the given event together.""" 96 | callback_gen = self.emit_routes_for(event.name, event.payload) 97 | callback_coros = (cb(event, *args, **kwargs) for cb in callback_gen) 98 | handler_tasks = map(asyncio.create_task, callback_coros) 99 | self._event_handler_tasks.update(handler_tasks) 100 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | # NOTE: "universal = 1" causes `bdist_wheel` to create a wheel that with the 3 | # NOTE: tag "py2.py3" which implies (and tricks pip into thinking) that this 4 | # NOTE: wheel contains Python 2 compatible code. This is not true and conflicts 5 | # NOTE: with the "Requires-Python" field in the metadata that says that we only 6 | # NOTE: support Python 3.6+. 7 | # NOTE: We need to keep it at "0" which will produce wheels tagged with "py3" 8 | # NOTE: when built under Python 3. 9 | # Ref: https://github.com/pypa/packaging.python.org/issues/726 10 | universal = 0 11 | 12 | [metadata] 13 | name = octomachinery 14 | url = https://octomachinery.dev 15 | project_urls = 16 | Chat: Matrix = https://matrix.to/#/#octomachinery:matrix.org 17 | Chat: Matrix (PyBA) = https://matrix.to/#/#pyba:matrix.org 18 | Chat: Matrix (@webknjaz) = https://matrix.to/#/@webknjaz:matrix.org 19 | CI: GitHub = https://github.com/sanitizers/octomachinery/actions/workflows/ci-cd.yml?query=event:push 20 | Docs: RTD = https://docs.octomachinery.dev 21 | GitHub: issues = https://github.com/sanitizers/octomachinery/issues 22 | GitHub: repo = https://github.com/sanitizers/octomachinery 23 | description = Invisible engine driving octobot machines. Simple, yet powerful. 24 | long_description = file: README.rst 25 | long_description_content_type = text/x-rst 26 | author = Sviatoslav Sydorenko (@webknjaz) 27 | author_email = wk+octomachinery@sydorenko.org.ua 28 | license = GPLv3+ 29 | license_file = LICENSE 30 | classifiers = 31 | Development Status :: 2 - Pre-Alpha 32 | 33 | Environment :: Console 34 | Environment :: Other Environment 35 | Environment :: Web Environment 36 | 37 | Framework :: AnyIO 38 | Framework :: AsyncIO 39 | 40 | Intended Audience :: Developers 41 | Intended Audience :: Information Technology 42 | Intended Audience :: System Administrators 43 | 44 | License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) 45 | 46 | Operating System :: OS Independent 47 | Operating System :: POSIX 48 | 49 | Programming Language :: Python 50 | Programming Language :: Python :: 3 51 | Programming Language :: Python :: 3.7 52 | Programming Language :: Python :: 3.8 53 | Programming Language :: Python :: 3.9 54 | Programming Language :: Python :: Implementation 55 | Programming Language :: Python :: Implementation :: CPython 56 | Programming Language :: Python :: Implementation :: PyPy 57 | 58 | Topic :: Internet :: WWW/HTTP 59 | Topic :: Internet :: WWW/HTTP :: HTTP Servers 60 | 61 | Topic :: Software Development 62 | Topic :: Software Development :: Libraries 63 | Topic :: Software Development :: Libraries :: Application Frameworks 64 | Topic :: Software Development :: Libraries :: Python Modules 65 | Topic :: Software Development :: Version Control 66 | Topic :: Software Development :: Version Control :: Git 67 | 68 | Topic :: System :: Networking 69 | 70 | Topic :: Utilities 71 | 72 | Typing :: Typed 73 | keywords = 74 | Bot 75 | Framework 76 | Framework for writing GitHub Apps 77 | GitHub 78 | GitHub Actions 79 | GitHub API 80 | GitHub Apps 81 | GitHub Checks API 82 | 83 | [options] 84 | python_requires = >=3.7 85 | package_dir = 86 | = . 87 | packages = find_namespace: 88 | zip_safe = True 89 | include_package_data = True 90 | 91 | # These are required in actual runtime: 92 | install_requires = 93 | aiohttp 94 | anyio < 2.0.0 95 | click 96 | cryptography 97 | environ-config >= 19.1.0 98 | envparse 99 | gidgethub >= 4.2.0 100 | pyjwt[crypto] 101 | pyyaml 102 | sentry_sdk 103 | setuptools_scm 104 | 105 | [options.packages.find] 106 | where = . 107 | 108 | [options.extras_require] 109 | docs = 110 | sphinx 111 | sphinxcontrib-apidoc 112 | furo 113 | testing = 114 | pytest 115 | pytest-aiohttp 116 | pytest-cov 117 | pytest-xdist 118 | -------------------------------------------------------------------------------- /octomachinery/github/api/raw_client.py: -------------------------------------------------------------------------------- 1 | """A very low-level GitHub API client.""" 2 | 3 | from asyncio import iscoroutinefunction 4 | from typing import Any, Dict, Optional, Tuple, Union 5 | 6 | from gidgethub.abc import JSON_CONTENT_TYPE 7 | from gidgethub.aiohttp import GitHubAPI 8 | 9 | # pylint: disable=relative-beyond-top-level 10 | from .tokens import GitHubJWTToken, GitHubOAuthToken, GitHubToken 11 | from .utils import accept_preview_version, mark_uninitialized_in_repr 12 | 13 | 14 | @mark_uninitialized_in_repr 15 | class RawGitHubAPI(GitHubAPI): 16 | """A low-level GitHub API client with a pre-populated token.""" 17 | 18 | def __init__( 19 | self, 20 | token: GitHubToken, 21 | *, 22 | user_agent: Optional[str] = None, 23 | **kwargs: Any, 24 | ) -> None: 25 | """Initialize the GitHub client with token.""" 26 | self._token = token 27 | kwargs.pop('oauth_token', None) 28 | kwargs.pop('jwt', None) 29 | super().__init__( 30 | requester=user_agent, 31 | **kwargs, 32 | ) 33 | 34 | @property 35 | def is_initialized(self): 36 | """Return GitHub token presence.""" 37 | return self._token is not None 38 | 39 | def __repr__(self): 40 | """Render a class instance representation.""" 41 | cls_name = self.__class__.__name__ 42 | init_args = ( 43 | f'token={self._token!r}, ' 44 | f'session={self._session!r}, ' 45 | f'user_agent={self.requester!r}' 46 | ) 47 | return f'{cls_name}({init_args})' 48 | 49 | # pylint: disable=arguments-differ 50 | # pylint: disable=keyword-arg-before-vararg 51 | # pylint: disable=too-many-arguments 52 | async def _make_request( 53 | self, method: str, url: str, url_vars: Dict[str, str], 54 | data: Any, accept: Union[str, None] = None, 55 | jwt: Optional[str] = None, 56 | oauth_token: Optional[str] = None, 57 | content_type: str = JSON_CONTENT_TYPE, 58 | extra_headers: Optional[Dict[str, str]] = None, 59 | ) -> Tuple[bytes, Optional[str]]: 60 | token = self._token 61 | if iscoroutinefunction(token): 62 | token = await token() 63 | if isinstance(token, GitHubOAuthToken): 64 | oauth_token = str(token) 65 | jwt = None 66 | if isinstance(token, GitHubJWTToken): 67 | jwt = str(token) 68 | oauth_token = None 69 | optional_kwargs = { 70 | # NOTE: GidgetHub v5.3.0 introduced a new `extra_headers` argument 71 | # NOTE: in this private method and the public ones. Its default 72 | # NOTE: value is `None` in all cases so the only case when it's set 73 | # NOTE: is when the end-users call corresponding methods with it. 74 | # NOTE: And that would only be the case with modern versions of 75 | # NOTE: GidgetHub. Here, we rely on this side effect to only pass 76 | # NOTE: this value down the stack when the chances that GidgetHub 77 | # NOTE: is modern enough are close to 100%. 78 | 'extra_headers': extra_headers, 79 | } if extra_headers is not None else {} 80 | return await super()._make_request( 81 | method=method, 82 | url=url, 83 | url_vars=url_vars, 84 | data=data, 85 | accept=accept, 86 | oauth_token=oauth_token, 87 | jwt=jwt, 88 | content_type=content_type, 89 | **optional_kwargs, 90 | ) 91 | 92 | getitem = accept_preview_version(GitHubAPI.getitem) 93 | getiter = accept_preview_version(GitHubAPI.getiter) 94 | post = accept_preview_version(GitHubAPI.post) 95 | patch = accept_preview_version(GitHubAPI.patch) 96 | put = accept_preview_version(GitHubAPI.put) 97 | delete = accept_preview_version(GitHubAPI.delete) 98 | -------------------------------------------------------------------------------- /octomachinery/github/models/__init__.py: -------------------------------------------------------------------------------- 1 | """Models representing objects in GitHub API.""" 2 | 3 | import typing 4 | from datetime import datetime, timezone 5 | 6 | import attr 7 | 8 | from .utils import SecretStr, convert_datetime 9 | 10 | 11 | @attr.dataclass 12 | class GitHubAppInstallation: # pylint: disable=too-few-public-methods 13 | """ 14 | Represents a GitHub App installed into a user or an organization profile. 15 | 16 | It has its own ID for installation which is a unique combo of an app 17 | and a profile (user or org). 18 | """ 19 | 20 | id: int = attr.ib(converter=int) 21 | """Installation ID.""" 22 | app_id: int = attr.ib(converter=int) 23 | """GitHub App ID.""" 24 | app_slug: str = attr.ib(converter=str) 25 | """GitHub App slug.""" 26 | 27 | # FIXME: unignore once this is solved: # pylint: disable=fixme 28 | # https://github.com/python/mypy/issues/6172#issuecomment-515718727 29 | created_at: datetime = attr.ib( 30 | converter=convert_datetime, # type: ignore[misc] 31 | ) 32 | """Date time when the installation has been installed.""" 33 | updated_at: datetime = attr.ib( 34 | converter=convert_datetime, # type: ignore[misc] 35 | ) 36 | """Date time when the installation was last updated.""" 37 | 38 | account: typing.Dict[str, typing.Any] 39 | """Target account (org or user) where this GitHub App is installed into.""" 40 | events: typing.List[str] 41 | """List of webhook events the app will be receiving from the account.""" 42 | permissions: typing.Dict[str, typing.Any] 43 | """Permission levels of access to API endpoints types.""" 44 | repository_selection: str = attr.ib(converter=str) 45 | """Repository selection mode.""" 46 | single_file_name: typing.Optional[str] 47 | """File path the GitHub app controls.""" 48 | 49 | target_id: int = attr.ib(converter=int) 50 | """Target account ID where this GitHub App is installed into.""" 51 | target_type: str = attr.ib( 52 | validator=attr.validators.in_(('Organization', 'User')), 53 | ) 54 | """Target account type where this GitHub App is installed into.""" 55 | 56 | access_tokens_url: str = attr.ib(converter=str) 57 | """API endpoint to retrieve access token from.""" 58 | html_url: str = attr.ib(converter=str) 59 | """URL for controlling the GitHub App Installation.""" 60 | repositories_url: str = attr.ib(converter=str) 61 | """API endpoint listing repositories accissible by this Installation.""" 62 | 63 | suspended_at: typing.Optional[str] 64 | suspended_by: typing.Optional[str] 65 | 66 | has_multiple_single_files: typing.Optional[bool] 67 | single_file_paths: typing.List[str] 68 | 69 | 70 | @attr.dataclass 71 | class GitHubInstallationAccessToken: # pylint: disable=too-few-public-methods 72 | """Struct for installation access token response from GitHub API.""" 73 | 74 | token: SecretStr = attr.ib(converter=SecretStr) 75 | """Access token for GitHub App Installation.""" 76 | expires_at: datetime = attr.ib( 77 | converter=convert_datetime, # type: ignore[misc] 78 | ) 79 | """Token expiration time.""" 80 | permissions: typing.Dict[str, str] 81 | """Permission levels of access to API endpoints types.""" 82 | repository_selection: str = attr.ib(converter=str) 83 | """Repository selection mode.""" 84 | repositories: typing.List[typing.Dict[str, typing.Any]] = attr.ib( 85 | default=[], 86 | converter=list, 87 | ) 88 | """List of accessible repositories.""" 89 | single_file: typing.Optional[str] = attr.ib(default=None) 90 | """File path the GitHub app controls.""" 91 | 92 | has_multiple_single_files: typing.Optional[bool] = attr.ib(default=False) 93 | single_file_paths: typing.List[str] = attr.ib(default=None) 94 | 95 | @property 96 | def expired(self): 97 | """Check whether this token has expired already.""" 98 | return datetime.now(timezone.utc) > self.expires_at 99 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # `git blame` master ignore list. 2 | # 3 | # This file contains a list of git hashes of revisions to be ignored 4 | # by `git blame`. These revisions are considered "unimportant" in 5 | # that they are unlikely to be what you are interested in when blaming. 6 | # They are typically expected to be formatting-only changes. 7 | # 8 | # It can be used for `git blame` using `--ignore-revs-file` or by 9 | # setting `blame.ignoreRevsFile` in the `git config`[1]. 10 | # 11 | # Ignore these commits when reporting with blame. Calling 12 | # 13 | # git blame --ignore-revs-file .git-blame-ignore-revs 14 | # 15 | # will tell `git blame` to ignore changes made by these revisions when 16 | # assigning blame, as if the change never happened. 17 | # 18 | # You can enable this as a default for your local repository by 19 | # running 20 | # 21 | # git config blame.ignoreRevsFile .git-blame-ignore-revs 22 | # 23 | # This will probably be automatically picked by your IDE 24 | # (VSCode+GitLens and JetBrains products are confirmed to do this). 25 | # 26 | # Important: if you are switching to a branch without this file, 27 | # `git blame` will fail with an error. 28 | # 29 | # GitHub also excludes the commits listed below from its "Blame" 30 | # views[2][3]. 31 | # 32 | # [1]: https://git-scm.com/docs/git-blame#Documentation/git-blame.txt-blameignoreRevsFile 33 | # [2]: https://github.blog/changelog/2022-03-24-ignore-commits-in-the-blame-view-beta/ 34 | # [3]: https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view 35 | # 36 | # Guidelines: 37 | # - Only large (generally automated) reformatting or renaming PRs 38 | # should be added to this list. Do not put things here just because 39 | # you feel they are trivial or unimportant. If in doubt, do not put 40 | # it on this list. 41 | # - When adding a single revision, use inline comment to link relevant 42 | # issue/PR. Alternatively, paste the commit title instead. 43 | # Example: 44 | # d4a8b7307acc2dc8a8833ccfa65426ad28b3ffc9 # https://github.com/sanitizers/octomachinery/issues/1 45 | # - When adding multiple revisions (like a bulk of work over many 46 | # commits), organize them in blocks. Precede each such block with a 47 | # comment starting with the word "START", followed by a link to the 48 | # relevant issue or PR. Add a similar comment after the last block 49 | # line but use the word "END", followed by the same link. 50 | # Alternatively, add or augment the link with a text motivation and 51 | # description of work performed in each commit. 52 | # After each individual commit in the block, add an inline comment 53 | # with the commit title line. 54 | # Example: 55 | # # START https://github.com/sanitizers/octomachinery/issues/1 56 | # 6f0bd2d8a1e6cd2e794cd39976e9756e0c85ac66 # Bulk-replace smile emojis with unicorns 57 | # d53974df11dbc22cbea9dc7dcbc9896c25979a27 # Replace double with single quotes 58 | # ... 59 | # # END https://github.com/sanitizers/octomachinery/issues/1 60 | # - Only put full 40-character hashes on this list (not short hashes 61 | # or any other revision reference). 62 | # - Append to the bottom of the file, regardless of the chronological 63 | # order of the revisions. Revisions within blocks should be in 64 | # chronological order from oldest to newest. 65 | # - Because you must use a hash, you need to append to this list in a 66 | # follow-up PR to the actual reformatting PR that you are trying to 67 | # ignore. This approach helps avoid issues with arbitrary rebases 68 | # and squashes while the pull request is in progress. 69 | 70 | 71 | 34dd2240adeb4d82e44a072dead0cc7197bc8f61 # A trailing comma reformatting caused by d3bee014f474d502d330c1c4919046c9714a6e4c ⇪ Bump add-trailing-comma @ pre-commit to v2.5.1 72 | 73 | # START yamllint-driven YAML formatting for consistency 74 | 8365edc0ef8e898aad55dab1a78b7dc2e4f4ee97 # 🎨 Fix yamllint leading space @ comment warnings 75 | 762437c1193a6b54c739edf4e62a999badbf9d75 # 🎨 Add YAML document start marker @ RTD config 76 | a7b255a6d44aa70dbb4062f76c7722c9dc02d0e7 # 🎨 Add YAML document end marker @ RTD config 77 | b1209aaa51e25e8b6ea88aa1682d1475382f31e7 # 🎨 Keep line length short @ GHA config 78 | # END yamllint-driven YAML formatting for consistency 79 | -------------------------------------------------------------------------------- /octomachinery/utils/asynctools.py: -------------------------------------------------------------------------------- 1 | """Asynchronous tools set.""" 2 | 3 | from functools import wraps 4 | from inspect import signature as _inspect_signature 5 | from logging import getLogger as _get_logger 6 | from operator import itemgetter 7 | 8 | from anyio import create_queue 9 | from anyio import create_task_group as all_subtasks_awaited 10 | 11 | 12 | logger = _get_logger(__name__) 13 | 14 | 15 | def auto_cleanup_aio_tasks(async_func): 16 | """Ensure all subtasks finish.""" 17 | @wraps(async_func) 18 | async def async_func_wrapper(*args, **kwargs): 19 | async with all_subtasks_awaited(): 20 | return await async_func(*args, **kwargs) 21 | return async_func_wrapper 22 | 23 | 24 | async def _send_task_res_to_q(res_q, task_id, aio_task): 25 | """Await task and put its result to the queue.""" 26 | try: 27 | task_res = await aio_task 28 | except (BaseException, Exception) as exc: 29 | task_res = exc 30 | raise 31 | finally: 32 | # pylint: disable-next=used-before-assignment # <-- false-positive 33 | await res_q.put((task_id, task_res)) 34 | 35 | 36 | async def _aio_gather_iter_pairs(*aio_tasks): 37 | """Spawn async tasks and yield with pairs of ids with results.""" 38 | aio_tasks_num = len(aio_tasks) 39 | task_res_q = create_queue(aio_tasks_num) 40 | 41 | async with all_subtasks_awaited() as task_group: 42 | for task_id, task in enumerate(aio_tasks): 43 | await task_group.spawn( 44 | _send_task_res_to_q, 45 | task_res_q, 46 | task_id, task, 47 | ) 48 | 49 | for _ in range(aio_tasks_num): 50 | yield await task_res_q.get() 51 | 52 | 53 | async def aio_gather_iter(*aio_tasks): 54 | """Spawn async tasks and yield results.""" 55 | async for _task_id, task_res in _aio_gather_iter_pairs(*aio_tasks): 56 | yield task_res 57 | 58 | 59 | async def aio_gather(*aio_tasks): 60 | """Spawn async tasks and return results in the same order.""" 61 | result_pairs_gen = [_r async for _r in _aio_gather_iter_pairs(*aio_tasks)] 62 | sorted_result_pairs = sorted(result_pairs_gen, key=itemgetter(0)) 63 | all_task_results = map(itemgetter(1), sorted_result_pairs) 64 | return tuple(all_task_results) 65 | 66 | 67 | async def try_await(potentially_awaitable): 68 | """Try awaiting the arg and return it regardless.""" 69 | valid_exc_str = ( 70 | "can't be used in 'await' expression" 71 | ) 72 | 73 | try: 74 | return await potentially_awaitable 75 | except TypeError as type_err: 76 | type_err_msg = str(type_err) 77 | if not ( 78 | type_err_msg.startswith('object ') 79 | and type_err_msg.endswith(valid_exc_str) 80 | ): 81 | raise 82 | 83 | return potentially_awaitable 84 | 85 | 86 | async def amap(callback, async_iterable): 87 | """Map asynchronous generator with a coroutine or a function.""" 88 | async for async_value in async_iterable: 89 | yield await try_await(callback(async_value)) 90 | 91 | 92 | def dict_to_kwargs_cb(callback): 93 | """Return a callback mapping dict to keyword arguments.""" 94 | cb_arg_names = set(_inspect_signature(callback).parameters.keys()) 95 | 96 | async def callback_wrapper(args_dict): 97 | excessive_arg_names = set(args_dict.keys()) - cb_arg_names 98 | filtered_args_dict = { 99 | arg_name: arg_value for arg_name, arg_value in args_dict.items() 100 | if arg_name not in excessive_arg_names 101 | } if excessive_arg_names else args_dict 102 | if excessive_arg_names: 103 | logger.warning( 104 | 'Excessive arguments passed to callback %(callable)s', 105 | {'callable': callback}, 106 | extra={ 107 | 'callable': callback, 108 | 'excessive-arg-names': excessive_arg_names, 109 | 'passed-in-args': args_dict, 110 | 'forwarded-args': filtered_args_dict, 111 | }, 112 | ) 113 | return await try_await(callback(**filtered_args_dict)) 114 | return callback_wrapper 115 | -------------------------------------------------------------------------------- /tests/github/models/private_key_test.py: -------------------------------------------------------------------------------- 1 | """Tests for GitHub private key class.""" 2 | import random 3 | import re 4 | from datetime import date 5 | from pathlib import Path 6 | 7 | import pytest 8 | 9 | from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat 10 | from jwt import decode as parse_jwt 11 | 12 | from octomachinery.github.models.private_key import GitHubPrivateKey 13 | 14 | 15 | @pytest.fixture 16 | def rsa_public_key(rsa_private_key): 17 | """Extract a public key out of private one.""" 18 | return rsa_private_key.public_key() 19 | 20 | 21 | @pytest.fixture 22 | def rsa_public_key_bytes(rsa_public_key) -> bytes: 23 | """Return a PKCS#1 formatted RSA public key encoded as PEM.""" 24 | return rsa_public_key.public_bytes( 25 | encoding=Encoding.PEM, 26 | format=PublicFormat.PKCS1, 27 | ) 28 | 29 | 30 | @pytest.fixture 31 | def github_private_key(rsa_private_key_bytes: bytes) -> GitHubPrivateKey: 32 | """Construct a test instance of ``GitHubPrivateKey``.""" 33 | return GitHubPrivateKey(rsa_private_key_bytes) 34 | 35 | 36 | @pytest.fixture 37 | def rsa_private_key_path( 38 | rsa_private_key_bytes: bytes, 39 | tmp_path_factory, 40 | ) -> Path: 41 | """Save the private key to disk as PEM file and provide its path.""" 42 | tmp_dir = tmp_path_factory.mktemp('github_private_key') 43 | private_key_filename = ( 44 | f'test-github-app.{date.today():%Y-%m-%d}' 45 | '.private-key.pem' 46 | ) 47 | private_key_path = tmp_dir / private_key_filename 48 | private_key_path.write_bytes(rsa_private_key_bytes) 49 | return private_key_path 50 | 51 | 52 | def test_github_private_key__from_file( 53 | github_private_key, 54 | rsa_private_key_path: Path, 55 | ): 56 | """Test that GitHubPrivateKey from file and bytes are the same.""" 57 | key_from_file = GitHubPrivateKey.from_file(rsa_private_key_path) 58 | assert key_from_file == github_private_key 59 | 60 | 61 | def test_github_private_key____repr__(github_private_key): 62 | """Verify what repr protocol only exposes fingerprint.""" 63 | repr_pattern = re.compile( 64 | r"^'\)\s" 65 | r'with\sSHA\-1\sfingerprint\s' 66 | r"'[a-f0-9]{2}(:[a-f0-9]{2}){19}'>$", 67 | ) 68 | assert repr_pattern.match(repr(github_private_key)) 69 | assert repr_pattern.match(f'{github_private_key!r}') 70 | 71 | 72 | def test_github_private_key____str__(github_private_key): 73 | """Verify that the string protocol doesn't expose secrets.""" 74 | escaped_private_key_repr = ( 75 | repr(github_private_key). 76 | replace('(', r'\('). 77 | replace(')', r'\)') 78 | ) 79 | exception_message = ( 80 | " " 81 | 'objects do not implement the string protocol ' 82 | 'for security reasons. ' 83 | f'The repr of this instance is {escaped_private_key_repr!s}.' 84 | ) 85 | with pytest.raises(TypeError, match=exception_message): 86 | str(github_private_key) 87 | with pytest.raises(TypeError, match=exception_message): 88 | f'{github_private_key!s}' # pylint: disable=pointless-statement 89 | 90 | 91 | def test_github_private_key__make_jwt_for( 92 | github_private_key: GitHubPrivateKey, 93 | rsa_public_key_bytes, 94 | ): 95 | """Verify that e2e encoding-decoding of the JWT works.""" 96 | github_app_id = random.randint(0, 9999999) 97 | jwt_string = github_private_key.make_jwt_for(app_id=github_app_id) 98 | payload = parse_jwt( 99 | jwt_string.encode('utf-8'), rsa_public_key_bytes, algorithms='RS256', 100 | ) 101 | assert payload['iss'] == github_app_id 102 | assert payload['exp'] - payload['iat'] == 60 103 | 104 | 105 | def test_github_private_key__make_jwt_for__invalid_timeout(github_private_key): 106 | """Verify that time offset can't exceed 10 mins.""" 107 | github_app_id = random.randint(0, 9999999) 108 | with pytest.raises( 109 | ValueError, 110 | match='The time offset must be less than 10 minutes', 111 | ): 112 | github_private_key.make_jwt_for(app_id=github_app_id, time_offset=601) 113 | -------------------------------------------------------------------------------- /tests/github/models/utils_test.py: -------------------------------------------------------------------------------- 1 | """Tests for GitHub models utility functions.""" 2 | 3 | from datetime import datetime, timezone 4 | from functools import partial 5 | 6 | import pytest 7 | 8 | from octomachinery.github.models.utils import ( 9 | SecretStr, SuperSecretStr, convert_datetime, 10 | ) 11 | 12 | 13 | # pylint: disable=invalid-name 14 | utc_datetime = partial(datetime, tzinfo=timezone.utc) 15 | 16 | 17 | @pytest.mark.parametrize( 18 | 'secret_class,secret_placeholder,visible_first_repr', 19 | ( 20 | (SecretStr, '', True), 21 | (SuperSecretStr, '', False), 22 | ), 23 | ) 24 | def test_secret_sanitizers_first_repr( 25 | secret_class, secret_placeholder, visible_first_repr, 26 | ): 27 | """Check that immediate repr is rendered correctly.""" 28 | secret_data = 'qwerty' 29 | super_secret_string = secret_class(secret_data) 30 | 31 | expected_repr = ( 32 | repr(secret_data) if visible_first_repr 33 | else secret_placeholder 34 | ) 35 | assert repr(super_secret_string) == expected_repr 36 | 37 | 38 | @pytest.mark.parametrize( 39 | 'secret_class,secret_placeholder', 40 | ( 41 | (SecretStr, ''), 42 | (SuperSecretStr, ''), 43 | ), 44 | ) 45 | def test_secret_sanitizers(secret_class, secret_placeholder): 46 | """Check that sanitizer classes hide data when needed.""" 47 | secret_data = 'qwerty' 48 | super_secret_string = secret_class(secret_data) 49 | assert str(super_secret_string) == str(secret_data) 50 | assert super_secret_string == secret_data 51 | 52 | class _DataStruct: # pylint: disable=too-few-public-methods 53 | def __init__(self, s, o=None): 54 | self._s = s 55 | self._o = o 56 | 57 | def __repr__(self): 58 | return ( 59 | f'{self.__class__.__name__}' 60 | f'(s={repr(self._s)}, o={repr(self._o)})' 61 | ) 62 | 63 | open_data = 'gov' 64 | data_struct = _DataStruct(super_secret_string, open_data) 65 | 66 | expected_data_struct_repr = ( 67 | f'_DataStruct(s={secret_placeholder}, o={repr(open_data)})' 68 | ) 69 | assert repr(data_struct) == expected_data_struct_repr 70 | 71 | sub_data_struct = _DataStruct(data_struct, secret_class(None)) 72 | 73 | expected_nested_data_struct_repr = ( 74 | f'_DataStruct(s=_DataStruct(s={secret_placeholder}, ' 75 | f'o={repr(open_data)}), o={secret_placeholder})' 76 | ) 77 | assert repr(sub_data_struct) == expected_nested_data_struct_repr 78 | 79 | 80 | @pytest.mark.parametrize( 81 | 'input_date_string,expected_date_object', 82 | ( 83 | (1556634127, utc_datetime(2019, 4, 30, 14, 22, 7)), 84 | (0, utc_datetime(1970, 1, 1, 0, 0)), 85 | ('2032-01-02T05:28:47Z', utc_datetime(2032, 1, 2, 5, 28, 47)), 86 | ('2032-01-02T05:28:47Z+00:00', utc_datetime(2032, 1, 2, 5, 28, 47)), 87 | ( 88 | '2032-01-02T05:28:47.000Z+00:00', 89 | utc_datetime(2032, 1, 2, 5, 28, 47), 90 | ), 91 | ( 92 | '2032-01-02T05:28:47.000000Z+00:00', 93 | utc_datetime(2032, 1, 2, 5, 28, 47), 94 | ), 95 | ('2032-01-02T05:28:47.000Z', utc_datetime(2032, 1, 2, 5, 28, 47)), 96 | ('2032-01-02T05:28:47.000000Z', utc_datetime(2032, 1, 2, 5, 28, 47)), 97 | ), 98 | ) 99 | def test_convert_datetime(input_date_string, expected_date_object): 100 | """Test that convert_datetime recognizes supported date formats.""" 101 | assert convert_datetime(input_date_string) == expected_date_object 102 | 103 | 104 | @pytest.mark.parametrize( 105 | 'input_date_string', 106 | ( 107 | None, 108 | float('NaN'), 109 | float('Inf'), 110 | -float('Inf'), 111 | object(), 112 | {}, 113 | set(), 114 | [], 115 | frozenset(), 116 | (), 117 | ), 118 | ) 119 | def test_convert_datetime_negative(input_date_string): 120 | """Test that convert_datetime errors out on supported date input.""" 121 | with pytest.raises( 122 | ValueError, 123 | match=r'^The input arg type .* is not supported$', 124 | ): 125 | convert_datetime(input_date_string) 126 | 127 | 128 | def test_convert_datetime_empty_string(): 129 | """Test that convert_datetime errors out on supported date input.""" 130 | with pytest.raises( 131 | ValueError, 132 | match=r'^The input arg .* is unsupported$', 133 | ): 134 | convert_datetime('') 135 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg 2 | :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md 3 | :alt: SWUbanner 4 | 5 | .. image:: https://img.shields.io/pypi/v/octomachinery.svg?logo=Python&logoColor=white 6 | :target: https://pypi.org/project/octomachinery 7 | :alt: octomachinery @ PyPI 8 | 9 | .. image:: https://tidelift.com/badges/package/pypi/octomachinery 10 | :target: https://tidelift.com/subscription/pkg/pypi-octomachinery?utm_source=pypi-octomachinery&utm_medium=readme 11 | :alt: octomachinery is available as part of the Tidelift Subscription 12 | 13 | .. image:: https://github.com/sanitizers/octomachinery/actions/workflows/ci-cd.yml/badge.svg?event=push 14 | :target: https://github.com/sanitizers/octomachinery/actions/workflows/ci-cd.yml?query=event:push 15 | :alt: GitHub Actions CI/CD workflows status 16 | 17 | .. image:: https://img.shields.io/matrix/octomachinery:matrix.org?label=Discuss%20on%20Matrix%20at%20%23octomachinery%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat 18 | :target: https://matrix.to/#/%23octomachinery:matrix.org 19 | :alt: Matrix Room — #octomachinery:matrix.org 20 | 21 | .. image:: https://img.shields.io/matrix/pyba:matrix.org?label=Discuss%20on%20Matrix%20at%20%23pyba%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat 22 | :target: https://matrix.to/#/%23pyba:matrix.org 23 | :alt: Matrix Space — #pyba:matrix.org 24 | 25 | .. DO-NOT-REMOVE-docs-badges-END 26 | 27 | .. image:: https://img.shields.io/readthedocs/octomachinery/latest.svg?logo=Read%20The%20Docs&logoColor=white 28 | :target: https://docs.octomachinery.dev/en/latest/?badge=latest 29 | :alt: Documentation Status 30 | 31 | octomachinery: Bots Without Boilerplate 32 | ======================================= 33 | 34 | Invisible engine driving octobot machines. Simple, yet powerful. 35 | 36 | Web-site @ https://octomachinery.dev. Stay tuned! 37 | 38 | .. DO-NOT-REMOVE-docs-intro-START 39 | 40 | **How-to create a GitHub Bot tutorial** is ready for preview 41 | @ `tutorial.octomachinery.dev 42 | `_ 43 | 44 | Elevator pitch 45 | -------------- 46 | 47 | Here's how you 👍 a just-created comment: 48 | 49 | .. code:: python 50 | 51 | from octomachinery.app.server.runner import run as run_app 52 | from octomachinery.routing import process_event_actions 53 | from octomachinery.routing.decorators import process_webhook_payload 54 | from octomachinery.runtime.context import RUNTIME_CONTEXT 55 | 56 | 57 | @process_event_actions('issue_comment', {'created'}) 58 | @process_webhook_payload 59 | async def on_comment( 60 | *, 61 | action, issue, comment, 62 | repository=None, sender=None, 63 | installation=None, 64 | assignee=None, changes=None, 65 | ): 66 | github_api = RUNTIME_CONTEXT.app_installation_client 67 | comment_reactions_api_url = f'{comment["url"]}/reactions' 68 | await github_api.post( 69 | comment_reactions_api_url, 70 | preview_api_version='squirrel-girl', 71 | data={'content': '+1'}, 72 | ) 73 | 74 | 75 | run_app( 76 | name='Thumbs-Up-Bot', 77 | version='1.0.0', 78 | url='https://github.com/apps/thuuuuuuuuuuuuuumbs-uuuuuuuuuuuup', 79 | ) 80 | 81 | Prerequisites 82 | ------------- 83 | 84 | Python 3.7+ 85 | 86 | Contribute octomachinery 87 | ------------------------ 88 | 89 | **Want to add something to upstream?** Feel free to submit a PR or file 90 | an issue if unsure. 91 | Note that PR is more likely to be accepted if it includes tests and 92 | detailed description helping maintainers to understand it better 🎉 93 | 94 | Oh, and be pythonic, please 🐍 95 | 96 | **Don't know how?** Check out `How to Contribute to Open Source 97 | `_ article by GitHub 🚀 98 | 99 | License 100 | ------- 101 | 102 | The source code and the documentation in this project are released under 103 | the `GPL v3 license`_. 104 | 105 | .. _`GPL v3 license`: 106 | https://github.com/sanitizers/octomachinery/blob/master/LICENSE 107 | 108 | For Enterprise 109 | -------------- 110 | 111 | octomachinery is available as part of the Tidelift Subscription. 112 | 113 | The octomachinery maintainers and the maintainers of thousands of other packages 114 | are working with Tidelift to deliver one enterprise subscription that covers 115 | all of the open source you use. 116 | 117 | `Learn more `_. 118 | -------------------------------------------------------------------------------- /octomachinery/github/utils/event_utils.py: -------------------------------------------------------------------------------- 1 | """Utility helpers for CLI.""" 2 | 3 | import contextlib 4 | import itertools 5 | import json 6 | from uuid import UUID, uuid4 7 | 8 | import multidict 9 | 10 | import yaml 11 | 12 | 13 | def _probe_yaml(event_file_fd): 14 | try: 15 | http_headers, event, extra = itertools.islice( 16 | itertools.chain( 17 | yaml.safe_load_all(event_file_fd), 18 | (None,) * 3, 19 | ), 20 | 3, 21 | ) 22 | except yaml.parser.ParserError as yaml_err: 23 | raise ValueError('YAML file is not valid') from yaml_err 24 | finally: 25 | event_file_fd.seek(0) 26 | 27 | if extra is not None: 28 | raise ValueError('YAML file must only contain 1–2 documents') 29 | 30 | if event is None: 31 | event = http_headers 32 | http_headers = () 33 | 34 | if event is None: 35 | raise ValueError('YAML file must contain 1–2 non-empty documents') 36 | 37 | return http_headers, event 38 | 39 | 40 | def _probe_jsonl(event_file_fd): 41 | event = None 42 | 43 | first_line = event_file_fd.readline() 44 | second_line = event_file_fd.readline() 45 | third_line = event_file_fd.readline() 46 | event_file_fd.seek(0) 47 | 48 | if third_line: 49 | raise ValueError('JSONL file must only contain 1–2 JSON lines') 50 | 51 | http_headers = json.loads(first_line) 52 | 53 | with contextlib.suppress(ValueError): 54 | event = json.loads(second_line) 55 | 56 | if event is None: 57 | event = http_headers 58 | http_headers = () 59 | 60 | return http_headers, event 61 | 62 | 63 | def _probe_json(event_file_fd): 64 | event = json.load(event_file_fd) 65 | event_file_fd.seek(0) 66 | 67 | if not isinstance(event, dict): 68 | raise ValueError('JSON file must only contain an object mapping') 69 | 70 | http_headers = () 71 | 72 | return http_headers, event 73 | 74 | 75 | def _parse_fd_content(event_file_fd): 76 | """Guess file content type and read event with HTTP headers.""" 77 | for event_reader in _probe_yaml, _probe_jsonl, _probe_json: 78 | with contextlib.suppress(ValueError): 79 | return event_reader(event_file_fd) 80 | 81 | raise ValueError( 82 | 'The input event VCR file has invalid structure. ' 83 | 'It must be either of YAML, JSONL or JSON.', 84 | ) 85 | 86 | 87 | def _transform_http_headers_list_to_multidict(headers): 88 | if isinstance(headers, dict): 89 | raise ValueError( 90 | 'Headers must be a sequence of mappings because keys can repeat', 91 | ) 92 | return multidict.CIMultiDict(next(iter(h.items()), ()) for h in headers) 93 | 94 | 95 | def parse_event_stub_from_fd(event_file_fd): 96 | """Read event with HTTP headers as CIMultiDict instance.""" 97 | http_headers, event = _parse_fd_content(event_file_fd) 98 | return _transform_http_headers_list_to_multidict(http_headers), event 99 | 100 | 101 | def validate_http_headers(headers): 102 | """Verify that HTTP headers look sane.""" 103 | if headers['content-type'] != 'application/json': 104 | raise ValueError("Content-Type must be 'application/json'") 105 | 106 | if not headers['user-agent'].startswith('GitHub-Hookshot/'): 107 | raise ValueError("User-Agent must start with 'GitHub-Hookshot/'") 108 | 109 | x_gh_delivery_exc = ValueError('X-GitHub-Delivery must be of type UUID4') 110 | try: 111 | x_gh_delivery_uuid = UUID(headers['x-github-delivery']) 112 | except ValueError as val_err: 113 | raise x_gh_delivery_exc from val_err 114 | if x_gh_delivery_uuid.version != 4: 115 | raise x_gh_delivery_exc 116 | 117 | if not isinstance(headers['x-github-event'], str): 118 | raise ValueError('X-GitHub-Event must be a string') 119 | 120 | 121 | def augment_http_headers(headers): 122 | """Add fake HTTP headers for the missing positions.""" 123 | fake_headers = make_http_headers_from_event(headers['x-github-event']) 124 | 125 | if 'content-type' not in headers: 126 | headers['content-type'] = fake_headers['content-type'] 127 | 128 | if 'user-agent' not in headers: 129 | headers['user-agent'] = fake_headers['user-agent'] 130 | 131 | if 'x-github-delivery' not in headers: 132 | headers['x-github-delivery'] = fake_headers['x-github-delivery'] 133 | 134 | return headers 135 | 136 | 137 | def make_http_headers_from_event(event_name): 138 | """Generate fake HTTP headers with the given event name.""" 139 | return multidict.CIMultiDict({ 140 | 'content-type': 'application/json', 141 | 'user-agent': 'GitHub-Hookshot/fallback-value', 142 | 'x-github-delivery': str(uuid4()), 143 | 'x-github-event': event_name, 144 | }) 145 | -------------------------------------------------------------------------------- /octomachinery/app/server/machinery.py: -------------------------------------------------------------------------------- 1 | """Web-server constructors.""" 2 | 3 | import functools 4 | import logging 5 | from typing import Union 6 | 7 | import anyio 8 | from aiohttp import web 9 | from aiohttp.client import ClientSession 10 | 11 | # pylint: disable=relative-beyond-top-level 12 | from ...github.api.app_client import GitHubApp 13 | # pylint: disable=relative-beyond-top-level 14 | from ...utils.asynctools import auto_cleanup_aio_tasks 15 | # pylint: disable=relative-beyond-top-level 16 | from ..routing.webhooks_dispatcher import route_github_webhook_event 17 | 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | async def start_tcp_site( 23 | server_config, aiohttp_server_runner: web.ServerRunner, 24 | ) -> web.TCPSite: 25 | """Return initialized and listening TCP site.""" 26 | host, port = server_config.host, server_config.port 27 | aiohttp_tcp_site = web.TCPSite(aiohttp_server_runner, host, port) 28 | await aiohttp_tcp_site.start() 29 | logger.info( 30 | ' Serving on %s '.center(50, '='), 31 | aiohttp_tcp_site.name, 32 | ) 33 | return aiohttp_tcp_site 34 | 35 | 36 | async def get_server_runner(http_handler): 37 | """Initialize server runner.""" 38 | aiohttp_server = web.Server(http_handler) 39 | aiohttp_server_runner = web.ServerRunner( 40 | aiohttp_server, 41 | # handle SIGTERM and SIGINT 42 | # by raising aiohttp.web_runner.GracefulExit exception 43 | handle_signals=True, 44 | ) 45 | await aiohttp_server_runner.setup() 46 | return aiohttp_server_runner 47 | 48 | 49 | async def _prepare_github_app(github_app): 50 | """Set GitHub App in the context.""" 51 | logger.info('Starting the following GitHub App:') 52 | logger.info( 53 | '* app id: %s', 54 | github_app._config.app_id, # pylint: disable=protected-access 55 | ) 56 | logger.info( 57 | '* private key SHA-1 fingerprint: %s', 58 | # pylint: disable=protected-access 59 | github_app._config.private_key.fingerprint, 60 | ) 61 | logger.info( 62 | '* user agent: %s', 63 | github_app._config.user_agent, # pylint: disable=protected-access 64 | ) 65 | await github_app.log_installs_list() 66 | 67 | 68 | async def _launch_web_server_and_wait_until_it_stops( 69 | web_server_config, 70 | github_app: GitHubApp, 71 | webhook_secret: Union[str, None] = None, 72 | ) -> None: 73 | """Start a web server. 74 | 75 | And then block until SIGINT comes in. 76 | """ 77 | aiohttp_server_runner = await setup_server_runner( 78 | github_app, webhook_secret, 79 | ) 80 | aiohttp_tcp_site = await start_tcp_site( 81 | web_server_config, aiohttp_server_runner, 82 | ) 83 | await _stop_site_on_cancel(aiohttp_tcp_site) 84 | 85 | 86 | async def setup_server_runner( 87 | github_app: GitHubApp, 88 | webhook_secret: Union[str, None] = None, 89 | ) -> web.ServerRunner: 90 | """Return a server runner with a webhook dispatcher set up.""" 91 | return await get_server_runner( 92 | functools.partial( 93 | route_github_webhook_event, 94 | github_app=github_app, 95 | webhook_secret=webhook_secret, 96 | ), 97 | ) 98 | 99 | 100 | async def _stop_site_on_cancel(aiohttp_tcp_site): 101 | """Stop the server after SIGINT.""" 102 | try: 103 | await anyio.sleep(float('inf')) 104 | except anyio.get_cancelled_exc_class(): 105 | logger.info(' Stopping the server '.center(50, '=')) 106 | await aiohttp_tcp_site.stop() 107 | 108 | 109 | def log_webhook_secret_status(webhook_secret): 110 | """Log HTTP body signature verification behavior.""" 111 | webhook_secret_repr = ( 112 | f' ({webhook_secret[:1]}...{webhook_secret[-1:]})' 113 | if webhook_secret else '' 114 | ) 115 | logger.info( 116 | 'Webhook secret%s is [%sSET]: %s', 117 | webhook_secret_repr, 118 | '' if webhook_secret else 'NOT ', 119 | 'SIGNATURE VERIFICATION WILL BE ENFORCED' 120 | if webhook_secret else 'SIGNED WEBHOOKS WILL BE REJECTED', 121 | ) 122 | 123 | 124 | @auto_cleanup_aio_tasks 125 | async def run_forever(config, event_routers): 126 | """Spawn an HTTP server in anyio context.""" 127 | logger.debug('The GitHub App env is set to `%s`', config.runtime.env) 128 | log_webhook_secret_status(config.github.webhook_secret) 129 | async with ClientSession() as aiohttp_client_session: 130 | github_app = GitHubApp( 131 | config.github, 132 | http_session=aiohttp_client_session, 133 | event_routers=event_routers, 134 | ) 135 | await _prepare_github_app(github_app) 136 | await _launch_web_server_and_wait_until_it_stops( 137 | config.server, github_app, config.github.webhook_secret, 138 | ) 139 | -------------------------------------------------------------------------------- /octomachinery/github/models/private_key.py: -------------------------------------------------------------------------------- 1 | """Private key container.""" 2 | from hashlib import sha1 as compute_sha1_hash 3 | from pathlib import Path 4 | from time import time 5 | 6 | from cryptography.hazmat.backends import default_backend 7 | from cryptography.hazmat.primitives.serialization import ( 8 | Encoding, PublicFormat, load_pem_private_key, 9 | ) 10 | 11 | from ._compat import compute_jwt 12 | 13 | 14 | def extract_private_key_sha1_fingerprint(rsa_private_key): 15 | r"""Retrieve the private key SHA-1 fingerprint. 16 | 17 | :param rsa_private_key: private key object 18 | :type rsa_private_key: cryptography.hazmat.primitives.asymmetric.\ 19 | rsa.RSAPrivateKey 20 | 21 | :returns: colon-separated SHA-1 fingerprint 22 | :rtype: str 23 | """ 24 | rsa_public_key = rsa_private_key.public_key() 25 | b_rsa_public_key = rsa_public_key.public_bytes( 26 | Encoding.DER, 27 | PublicFormat.SubjectPublicKeyInfo, 28 | ) 29 | rsa_public_key_sha1_fingerprint = compute_sha1_hash( 30 | b_rsa_public_key, 31 | ).hexdigest() 32 | 33 | def emit_chunks(sequence, step): 34 | start_pos = 0 35 | seq_length = len(sequence) 36 | while start_pos < seq_length: 37 | end_pos = start_pos + step 38 | yield sequence[start_pos: end_pos] 39 | start_pos = end_pos 40 | 41 | return ':'.join( 42 | emit_chunks(rsa_public_key_sha1_fingerprint, 2), 43 | ) 44 | 45 | 46 | class GitHubPrivateKey: 47 | """Private key entity with a pre-calculated SHA-1 fingerprint. 48 | 49 | :param bytes b_raw_data: the contents of a PEM file 50 | """ 51 | 52 | def __init__(self, b_raw_data: bytes): 53 | """Initialize GitHubPrivateKey instance.""" 54 | self._rsa_private_key = load_pem_private_key( 55 | b_raw_data, 56 | password=None, 57 | backend=default_backend(), 58 | ) 59 | self._col_separated_rsa_public_key_sha1_fingerprint = ( 60 | extract_private_key_sha1_fingerprint(self._rsa_private_key) 61 | ) 62 | 63 | @property 64 | def fingerprint(self) -> str: 65 | """Colon-separated SHA-1 fingerprint string value. 66 | 67 | :returns: colon-separated SHA-1 fingerprint 68 | :rtype: str 69 | """ 70 | return self._col_separated_rsa_public_key_sha1_fingerprint 71 | 72 | def __str__(self): 73 | """Avoid leaking private key contents via string protocol. 74 | 75 | :raises TypeError: always 76 | """ 77 | raise TypeError( 78 | f'{type(self)} objects do not implement the string protocol ' 79 | 'for security reasons. ' 80 | f'The repr of this instance is {self!r}.', 81 | ) 82 | 83 | def __repr__(self): 84 | r"""Construct a GitHubPrivateKey object representation. 85 | 86 | :returns: GitHubPrivateKey object representation \ 87 | with its SHA-1 fingerprint 88 | :rtype: str 89 | """ 90 | return ( 91 | "') " 92 | f"with SHA-1 fingerprint '{self.fingerprint}'>" 93 | ) 94 | 95 | def __eq__(self, other_private_key): 96 | r"""Compare equality of our private key with other. 97 | 98 | :returns: the result of comparison with another \ 99 | ``GitHubPrivateKey`` instance 100 | :rtype: bool 101 | """ 102 | return self.matches_fingerprint(other_private_key.fingerprint) 103 | 104 | def matches_fingerprint(self, other_hash): 105 | """Compare our SHA-1 fingerprint with ``other_hash``. 106 | 107 | :returns: the result of own fingerprint comparison with ``other_hash`` 108 | :rtype: bool 109 | """ 110 | return self.fingerprint == other_hash 111 | 112 | @classmethod 113 | def from_file(cls, path): 114 | r"""Construct a ``GitHubPrivateKey`` instance. 115 | 116 | :returns: the ``GitHubPrivateKey`` instance \ 117 | constructed of the target file contents 118 | :rtype: GitHubPrivateKey 119 | """ 120 | return cls(Path(path).expanduser().read_bytes()) 121 | 122 | def make_jwt_for(self, *, app_id: int, time_offset: int = 60) -> str: 123 | r"""Generate app's JSON Web Token. 124 | 125 | :param int app_id: numeric ID of a GitHub App 126 | :param int time_offset: duration of the JWT's validity, in seconds, \ 127 | defaults to 60 128 | 129 | :returns: JWT string for a GitHub App valid for the given time 130 | :rtype: str 131 | 132 | :raises ValueError: if time_offset exceeds 600 seconds (10 minutes) 133 | """ 134 | ten_min = 60 * 10 135 | if time_offset > ten_min: 136 | raise ValueError('The time offset must be less than 10 minutes') 137 | 138 | now = int(time()) 139 | payload = { 140 | 'iat': now, 141 | 'exp': now + time_offset, 142 | 'iss': app_id, 143 | } 144 | 145 | return compute_jwt( 146 | payload, 147 | key=self._rsa_private_key, 148 | algorithm='RS256', 149 | ) 150 | -------------------------------------------------------------------------------- /octomachinery/app/routing/webhooks_dispatcher.py: -------------------------------------------------------------------------------- 1 | """GitHub webhook events dispatching logic.""" 2 | 3 | import asyncio 4 | import logging 5 | import typing 6 | from functools import wraps 7 | from http import HTTPStatus 8 | 9 | from aiohttp import web 10 | from gidgethub import BadRequest, ValidationFailure 11 | from gidgethub.sansio import validate_event as validate_webhook_payload 12 | 13 | # pylint: disable=relative-beyond-top-level,import-error 14 | from ...github.models.events import GidgetHubWebhookEvent 15 | # pylint: disable=relative-beyond-top-level,import-error 16 | from ...routing.webhooks_dispatcher import route_github_event 17 | 18 | 19 | __all__ = ('route_github_webhook_event',) 20 | 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | EVENT_LOG_TMPL = ( 26 | 'Got a{}valid X-GitHub-Event=%s ' 27 | 'with X-GitHub-Delivery=%s ' 28 | 'and X-Hub-Signature=%s' 29 | ) 30 | 31 | EVENT_INVALID_CHUNK = 'n in' 32 | EVENT_VALID_CHUNK = ' ' 33 | 34 | EVENT_LOG_VALID_MSG = EVENT_LOG_TMPL.format(EVENT_VALID_CHUNK) 35 | EVENT_LOG_INVALID_MSG = EVENT_LOG_TMPL.format(EVENT_INVALID_CHUNK) 36 | 37 | 38 | async def get_trusted_http_payload(request, webhook_secret): 39 | """Get a verified HTTP request body from request.""" 40 | http_req_headers = request.headers 41 | is_secret_provided = webhook_secret is not None 42 | is_payload_signed = 'x-hub-signature' in http_req_headers 43 | 44 | if is_payload_signed and not is_secret_provided: 45 | raise ValidationFailure('secret not provided') 46 | 47 | if not is_payload_signed and is_secret_provided: 48 | raise ValidationFailure('signature is missing') 49 | 50 | raw_http_req_body = await request.read() 51 | 52 | if is_payload_signed and is_secret_provided: 53 | validate_webhook_payload( 54 | payload=raw_http_req_body, 55 | signature=http_req_headers['x-hub-signature'], 56 | secret=webhook_secret, 57 | ) 58 | 59 | return raw_http_req_body 60 | 61 | 62 | async def get_event_from_request(request, webhook_secret): 63 | """Retrieve Event out of HTTP request if it's valid.""" 64 | webhook_event_signature = request.headers.get( 65 | 'X-Hub-Signature', '', 66 | ) 67 | try: 68 | http_req_body = await get_trusted_http_payload( 69 | request, webhook_secret, 70 | ) 71 | except ValidationFailure as no_signature_exc: 72 | logger.error( 73 | EVENT_LOG_INVALID_MSG, 74 | request.headers.get('X-GitHub-Event'), 75 | request.headers.get('X-GitHub-Delivery'), 76 | webhook_event_signature, 77 | ) 78 | logger.debug( 79 | 'Webhook HTTP query signature validation failed because: %s', 80 | no_signature_exc, 81 | ) 82 | raise web.HTTPForbidden from no_signature_exc 83 | 84 | event = GidgetHubWebhookEvent.from_http_request( 85 | http_req_headers=request.headers, 86 | http_req_body=http_req_body, 87 | ) 88 | logger.info( 89 | EVENT_LOG_VALID_MSG, 90 | event.name, # pylint: disable=no-member 91 | event.delivery_id, 92 | webhook_event_signature, 93 | ) 94 | return event 95 | 96 | 97 | def validate_allowed_http_methods(*allowed_methods: str): 98 | """Block disallowed HTTP methods.""" 99 | _allowed_methods: typing.Set[str] 100 | if not allowed_methods: 101 | _allowed_methods = {'POST'} 102 | else: 103 | _allowed_methods = set(allowed_methods) 104 | 105 | def decorator(wrapped_function): 106 | @wraps(wrapped_function) 107 | async def wrapper(request, *, github_app, webhook_secret=None): 108 | if request.method not in _allowed_methods: 109 | raise web.HTTPMethodNotAllowed( 110 | method=request.method, 111 | allowed_methods=_allowed_methods, 112 | ) from BadRequest(HTTPStatus.METHOD_NOT_ALLOWED) 113 | return await wrapped_function( 114 | request, 115 | github_app=github_app, 116 | webhook_secret=webhook_secret, 117 | ) 118 | return wrapper 119 | return decorator 120 | 121 | 122 | def webhook_request_to_event(wrapped_function): 123 | """Pass event extracted from request into the wrapped function.""" 124 | @wraps(wrapped_function) 125 | async def wrapper(request, *, github_app, webhook_secret=None): 126 | event = await get_event_from_request(request, webhook_secret) 127 | return await wrapped_function( 128 | github_event=event, github_app=github_app, 129 | ) 130 | return wrapper 131 | 132 | 133 | @validate_allowed_http_methods('POST') 134 | @webhook_request_to_event 135 | async def route_github_webhook_event(*, github_event, github_app): 136 | """Dispatch incoming webhook events to corresponding handlers.""" 137 | asyncio.create_task( 138 | route_github_event( 139 | github_event=github_event, 140 | github_app=github_app, 141 | ), 142 | ) 143 | event_ack_msg = ( 144 | 'GitHub event received and scheduled for processing. ' 145 | f'It is {github_event!r}' 146 | ) 147 | return web.Response(text=f'OK: {event_ack_msg!s}') 148 | -------------------------------------------------------------------------------- /tests/app/server/machinery_test.py: -------------------------------------------------------------------------------- 1 | """Test app server machinery.""" 2 | 3 | import uuid 4 | from typing import Tuple 5 | 6 | from aiohttp.client import ClientSession 7 | from aiohttp.test_utils import get_unused_port_socket 8 | from aiohttp.web import SockSite 9 | 10 | import pytest 11 | 12 | from octomachinery.app.config import BotAppConfig 13 | from octomachinery.app.routing import WEBHOOK_EVENTS_ROUTER 14 | from octomachinery.app.server.machinery import setup_server_runner 15 | from octomachinery.github.api.app_client import GitHubApp 16 | 17 | 18 | IPV4_LOCALHOST = '127.0.0.1' 19 | 20 | 21 | @pytest.fixture 22 | def ephemeral_port_tcp_sock(): 23 | """Initialize an ephemeral TCP socket.""" 24 | return get_unused_port_socket(IPV4_LOCALHOST) 25 | 26 | 27 | @pytest.fixture 28 | def ephemeral_port_tcp_sock_addr(ephemeral_port_tcp_sock): 29 | """Return final host and port addr of the ephemeral TCP socket.""" 30 | return ephemeral_port_tcp_sock.getsockname()[:2] 31 | 32 | 33 | @pytest.fixture 34 | def github_app_id() -> int: 35 | """Return a GitHub App ID.""" 36 | return 0 37 | 38 | 39 | @pytest.fixture 40 | def octomachinery_config( 41 | github_app_id: int, rsa_private_key_bytes: bytes, 42 | ephemeral_port_tcp_sock_addr: Tuple[str, int], 43 | ) -> None: 44 | """Initialize a GitHub App bot config.""" 45 | host, port = ephemeral_port_tcp_sock_addr 46 | # https://github.com/hynek/environ-config/blob/master/CHANGELOG.rst#1910-2019-09-02 47 | # pylint: disable=no-member 48 | return BotAppConfig.from_environ({ # type: ignore[attr-defined] 49 | 'GITHUB_APP_IDENTIFIER': str(github_app_id), 50 | 'GITHUB_PRIVATE_KEY': rsa_private_key_bytes.decode(), 51 | 'HOST': host, 52 | 'PORT': port, 53 | }) 54 | 55 | 56 | @pytest.fixture 57 | def octomachinery_config_github_app(octomachinery_config): 58 | """Return a GitHub App bot config section.""" 59 | return octomachinery_config.github 60 | 61 | 62 | @pytest.fixture 63 | def octomachinery_config_server(octomachinery_config): 64 | """Return a GitHub App server config section.""" 65 | return octomachinery_config.server 66 | 67 | 68 | @pytest.fixture 69 | async def aiohttp_client_session(event_loop) -> ClientSession: 70 | """Initialize an aiohttp HTTP client session.""" 71 | async with ClientSession() as http_session: 72 | yield http_session 73 | 74 | 75 | @pytest.fixture 76 | def octomachinery_event_routers(): 77 | """Construct a set of routers for use in the GitHub App.""" 78 | return frozenset({WEBHOOK_EVENTS_ROUTER}) 79 | 80 | 81 | @pytest.fixture 82 | def github_app( 83 | octomachinery_config_github_app, aiohttp_client_session, 84 | octomachinery_event_routers, 85 | ): 86 | """Initizalize a GitHub App instance.""" 87 | return GitHubApp( 88 | octomachinery_config_github_app, 89 | aiohttp_client_session, 90 | octomachinery_event_routers, 91 | ) 92 | 93 | 94 | @pytest.fixture 95 | async def octomachinery_app_server_runner(github_app): 96 | """Set up an HTTP handler for webhooks.""" 97 | return await setup_server_runner(github_app) 98 | 99 | 100 | @pytest.fixture 101 | async def octomachinery_app_tcp( 102 | ephemeral_port_tcp_sock, 103 | octomachinery_app_server_runner, 104 | ): 105 | """Run octomachinery web server and tear-down after testing.""" 106 | tcp_site = SockSite( 107 | octomachinery_app_server_runner, ephemeral_port_tcp_sock, 108 | ) 109 | await tcp_site.start() 110 | try: 111 | yield tcp_site 112 | finally: 113 | await tcp_site.stop() 114 | 115 | 116 | @pytest.fixture 117 | async def send_webhook_event( 118 | octomachinery_app_tcp, aiohttp_client_session, 119 | ): 120 | """Return a webhook sender coroutine.""" 121 | def _send_event(webhook_payload=None): 122 | post_body = {} if webhook_payload is None else webhook_payload 123 | 124 | webhook_endpoint_url = octomachinery_app_tcp.name 125 | return aiohttp_client_session.post( 126 | webhook_endpoint_url, json=post_body, 127 | headers={ 128 | 'X-GitHub-Delivery': str(uuid.uuid4()), 129 | 'X-GitHub-Event': 'ping', 130 | }, 131 | ) 132 | return _send_event 133 | 134 | 135 | @pytest.mark.anyio 136 | async def test_ping_response(send_webhook_event, github_app_id): 137 | """Test that ping webhook event requests receive a HTTP response.""" 138 | async with send_webhook_event( 139 | { 140 | 'hook': {'app_id': github_app_id}, 141 | 'hook_id': 0, 142 | 'zen': 'Hey zen!', 143 | }, 144 | ) as gh_app_http_resp: 145 | resp_body: str = (await gh_app_http_resp.read()).decode() 146 | 147 | resp_content_type = gh_app_http_resp.headers['Content-Type'] 148 | expected_response_start = ( 149 | 'OK: GitHub event received and scheduled for processing. ' 150 | "It is GidgetHubWebhookEvent(name='ping', payload={'hook': {" 151 | f"'app_id': {github_app_id}" 152 | "}, 'hook_id': 0, 'zen': 'Hey zen!'}, delivery_id=UUID('" 153 | ) 154 | 155 | assert resp_content_type == 'text/plain; charset=utf-8' 156 | assert resp_body.startswith(expected_response_start) 157 | -------------------------------------------------------------------------------- /tests/circular_imports_test.py: -------------------------------------------------------------------------------- 1 | """Tests for circular imports in all local packages and modules. 2 | 3 | This ensures all internal packages can be imported right away without 4 | any need to import some other module before doing so. 5 | 6 | This module is based on an idea that pytest uses for self-testing: 7 | * https://github.com/pytest-dev/pytest/blob/d18c75b/testing/test_meta.py 8 | * https://twitter.com/codewithanthony/status/1229445110510735361 9 | """ 10 | import os 11 | import pkgutil 12 | import subprocess 13 | import sys 14 | from itertools import chain 15 | from pathlib import Path 16 | 17 | import pytest 18 | 19 | import octomachinery 20 | 21 | 22 | def _find_all_importables(pkg): 23 | """Find all importables in the project. 24 | 25 | Return them in order. 26 | """ 27 | return sorted( 28 | set( 29 | chain.from_iterable( 30 | _discover_path_importables(Path(p), pkg.__name__) 31 | for p in pkg.__path__ 32 | ), 33 | ), 34 | ) 35 | 36 | 37 | def _discover_path_importables(pkg_pth, pkg_name): 38 | """Yield all importables under a given path and package.""" 39 | for dir_path, _d, file_names in os.walk(pkg_pth): 40 | pkg_dir_path = Path(dir_path) 41 | 42 | if pkg_dir_path.parts[-1] == '__pycache__': 43 | continue 44 | 45 | if all(Path(_).suffix != '.py' for _ in file_names): 46 | continue 47 | 48 | rel_pt = pkg_dir_path.relative_to(pkg_pth) 49 | pkg_pref = '.'.join((pkg_name,) + rel_pt.parts) 50 | yield from ( 51 | pkg_path 52 | for _, pkg_path, _ in pkgutil.walk_packages( 53 | (str(pkg_dir_path),), prefix=f'{pkg_pref}.', 54 | ) 55 | ) 56 | 57 | 58 | @pytest.mark.parametrize( 59 | 'import_path', 60 | _find_all_importables(octomachinery), 61 | ) 62 | def test_no_warnings(import_path): 63 | """Verify that exploding importables doesn't explode. 64 | 65 | This is seeking for any import errors including ones caused 66 | by circular imports. 67 | """ 68 | imp_cmd = ( 69 | sys.executable, 70 | '-W', 'error', 71 | 72 | # NOTE: These are necessary for `tox -e old-deps`: 73 | '-W', "ignore:Using or importing the ABCs from 'collections' instead " 74 | "of from 'collections.abc' is deprecated since " 75 | 'Python 3.3, and in 3.10 it will stop working:' 76 | 'DeprecationWarning:jwt.api_jwt', 77 | '-W', "ignore:Using or importing the ABCs from 'collections' instead " 78 | "of from 'collections.abc' is deprecated since " 79 | 'Python 3.3, and in 3.9 it will stop working:' 80 | 'DeprecationWarning:jwt.api_jwt', 81 | # NOTE: This looks like the line above but has a typo 82 | # NOTE: (a whitespace is missing): 83 | '-W', "ignore:Using or importing the ABCs from 'collections' instead " 84 | "of from 'collections.abc' is deprecated since " 85 | 'Python 3.3,and in 3.9 it will stop working:' 86 | 'DeprecationWarning:jwt.api_jwt', 87 | 88 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 89 | # NOTE: command via `tox -e old-deps`: 90 | '-W', 'ignore:pkg_resources is deprecated as an API:' 91 | 'DeprecationWarning:pkg_resources', 92 | 93 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 94 | # NOTE: command via `tox -e old-deps`: 95 | '-W', 'ignore:The distutils package is deprecated and slated for ' 96 | 'removal in Python 3.12. Use setuptools or check PEP 632 for ' 97 | 'potential alternatives:' 98 | 'DeprecationWarning:', 99 | 100 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 101 | # NOTE: command via `tox -e old-deps`: 102 | '-W', "ignore:module 'sre_constants' is deprecated:" 103 | 'DeprecationWarning:', 104 | 105 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 106 | # NOTE: command via `tox -e old-deps`: 107 | '-W', 'ignore:_SixMetaPathImporter.exec_module() not found; ' 108 | 'falling back to load_module():ImportWarning:', 109 | 110 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 111 | # NOTE: command via `tox -e old-deps`: 112 | '-W', 'ignore:_SixMetaPathImporter.find_spec() not found; ' 113 | 'falling back to find_module():ImportWarning:', 114 | 115 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 116 | # NOTE: command via `tox -e old-deps`: 117 | '-W', 'ignore:VendorImporter.exec_module() not found; ' 118 | 'falling back to load_module():ImportWarning:', 119 | 120 | # NOTE: Triggered by the `octomachinery.utils.versiontools` 121 | # NOTE: command via `tox -e old-deps`: 122 | '-W', 'ignore:VendorImporter.find_spec() not found; ' 123 | 'falling back to find_module():ImportWarning:', 124 | 125 | # NOTE: Triggered by the `octomachinery.routing.routers` 126 | # NOTE: command via `tox -e old-deps`: 127 | '-W', "ignore:'cgi' is deprecated and slated for removal " 128 | 'in Python 3.13:' 129 | 'DeprecationWarning:gidgethub.sansio', 130 | 131 | '-W', 'ignore:"@coroutine" decorator is deprecated ' 132 | 'since Python 3.8, use "async def" instead:' 133 | 'DeprecationWarning:aiohttp.helpers', 134 | '-c', f'import {import_path!s}', 135 | ) 136 | 137 | subprocess.check_call(imp_cmd) 138 | -------------------------------------------------------------------------------- /octomachinery/github/api/app_client.py: -------------------------------------------------------------------------------- 1 | """GitHub App API client.""" 2 | 3 | from __future__ import annotations 4 | 5 | import logging 6 | from collections import defaultdict 7 | from typing import TYPE_CHECKING, Any, Dict, Iterable 8 | 9 | from aiohttp.client import ClientSession 10 | from aiohttp.client_exceptions import ClientConnectorError 11 | 12 | import attr 13 | import sentry_sdk 14 | 15 | # pylint: disable=relative-beyond-top-level 16 | from ...routing import WEBHOOK_EVENTS_ROUTER 17 | # pylint: disable=relative-beyond-top-level 18 | from ...utils.asynctools import amap, dict_to_kwargs_cb 19 | # pylint: disable=relative-beyond-top-level 20 | from ..config.app import GitHubAppIntegrationConfig 21 | # pylint: disable=relative-beyond-top-level 22 | from ..entities.app_installation import GitHubAppInstallation 23 | # pylint: disable=relative-beyond-top-level 24 | from ..models import GitHubAppInstallation as GitHubAppInstallationModel 25 | # pylint: disable=relative-beyond-top-level 26 | from ..models.events import GitHubEvent 27 | from .raw_client import RawGitHubAPI 28 | from .tokens import GitHubJWTToken 29 | 30 | 31 | if TYPE_CHECKING: 32 | # pylint: disable=relative-beyond-top-level 33 | from ...routing.abc import OctomachineryRouterBase 34 | 35 | 36 | logger = logging.getLogger(__name__) 37 | 38 | 39 | GH_INSTALL_EVENTS = {'integration_installation', 'installation'} 40 | 41 | 42 | @attr.dataclass 43 | class GitHubApp: 44 | """GitHub API wrapper.""" 45 | 46 | _config: GitHubAppIntegrationConfig 47 | _http_session: ClientSession 48 | _event_routers: Iterable[OctomachineryRouterBase] = attr.ib( 49 | default={WEBHOOK_EVENTS_ROUTER}, 50 | converter=frozenset, 51 | ) 52 | 53 | def __attrs_post_init__(self) -> None: 54 | """Initialize the Sentry SDK library.""" 55 | # NOTE: Under the hood, it will set up the DSN from `SENTRY_DSN` 56 | # NOTE: env var. We don't need to care about it not existing as 57 | # NOTE: Sentry SDK helpers don't fail loudly and if not 58 | # NOTE: configured, it'll be ignored. 59 | # FIXME: # pylint: disable=fixme 60 | sentry_sdk.init() # pylint: disable=abstract-class-instantiated 61 | 62 | async def dispatch_event(self, github_event: GitHubEvent) -> Iterable[Any]: 63 | """Dispatch ``github_event`` into the embedded routers.""" 64 | return await github_event.dispatch_via( 65 | *self._event_routers, # pylint: disable=not-an-iterable 66 | ) 67 | 68 | async def log_installs_list(self) -> None: 69 | """Store all installations data before starting.""" 70 | try: 71 | installations = await self.get_installations() 72 | except ClientConnectorError as client_error: 73 | logger.info('It looks like the GitHub API is offline...') 74 | logger.error( 75 | 'The following error has happened while trying to grab ' 76 | 'installations list: %s', 77 | client_error, 78 | ) 79 | return 80 | 81 | logger.info('This GitHub App is installed into:') 82 | # pylint: disable=protected-access 83 | for install_id, install_val in installations.items(): 84 | logger.info( 85 | '* Installation id %s (installed to %s)', 86 | install_id, 87 | install_val._metadata.account['login'], 88 | ) 89 | 90 | @property 91 | def gh_jwt(self): 92 | """Generate app's JSON Web Token, valid for 60 seconds.""" 93 | token = self._config.private_key.make_jwt_for( 94 | app_id=self._config.app_id, 95 | ) 96 | return GitHubJWTToken(token) 97 | 98 | @property 99 | def api_client(self): # noqa: D401 100 | """The GitHub App client with an async CM interface.""" 101 | return RawGitHubAPI( 102 | token=self.gh_jwt, 103 | session=self._http_session, 104 | user_agent=self._config.user_agent, 105 | ) 106 | 107 | async def get_installation(self, event): 108 | """Retrieve an installation creds from store.""" 109 | if 'installation' not in event.payload: 110 | raise LookupError('This event occurred outside of an installation') 111 | 112 | install_id = event.payload['installation']['id'] 113 | return await self.get_installation_by_id(install_id) 114 | 115 | async def get_installation_by_id(self, install_id): 116 | """Retrieve an installation with access tokens via API.""" 117 | return GitHubAppInstallation( 118 | await dict_to_kwargs_cb(GitHubAppInstallationModel)( 119 | await self.api_client.getitem( 120 | '/app/installations/{installation_id}', 121 | url_vars={'installation_id': install_id}, 122 | preview_api_version='machine-man', 123 | ), 124 | ), 125 | self, 126 | ) 127 | 128 | async def get_installations(self): 129 | """Retrieve all installations with access tokens via API.""" 130 | installations: Dict[ 131 | int, GitHubAppInstallation, 132 | ] = defaultdict(dict) # type: ignore[arg-type] 133 | async for install in amap( 134 | dict_to_kwargs_cb(GitHubAppInstallationModel), 135 | self.api_client.getiter( 136 | '/app/installations', 137 | preview_api_version='machine-man', 138 | ), 139 | ): 140 | installations[install.id] = GitHubAppInstallation( 141 | install, self, 142 | ) 143 | return installations 144 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | 3 | # Print the total number of errors: 4 | count = true 5 | 6 | # Don't even try to analyze these: 7 | extend-exclude = 8 | # Circle CI configs 9 | .circleci, 10 | # No need to traverse egg info dir 11 | *.egg-info, 12 | # GitHub configs 13 | .github, 14 | # Cache files of MyPy 15 | .mypy_cache, 16 | # Cache files of pytest 17 | .pytest_cache, 18 | # Temp dir of pytest-testmon 19 | .tmontmp, 20 | # Countless third-party libs in venvs 21 | .tox, 22 | # Occasional virtualenv dir 23 | .venv, 24 | # VS Code 25 | .vscode, 26 | # Temporary build dir 27 | build, 28 | # This contains sdists and wheels that we don't want to check 29 | dist, 30 | # Metadata of `pip wheel` cmd is autogenerated 31 | pip-wheel-metadata, 32 | 33 | # IMPORTANT: avoid using ignore option, always use extend-ignore instead 34 | # Completely and unconditionally ignore the following errors: 35 | extend-ignore = 36 | # Legitimate cases, no need to "fix" these violations: 37 | # E501: "line too long", its function is replaced by `flake8-length` 38 | E501, 39 | # W505: "doc line too long", its function is replaced by `flake8-length` 40 | W505, 41 | # I: flake8-isort is drunk + we have isort integrated into pre-commit 42 | # I, 43 | # WPS300: "Found local folder import" -- nothing bad about this 44 | WPS300, 45 | # WPS305: "Found f string" -- nothing bad about this 46 | WPS305, 47 | # An opposite consistency expectation is currently enforced 48 | # by pylint via: useless-object-inheritance (R0205): 49 | # WPS306: "Found class without a base class: *" -- we have metaclass shims 50 | WPS306, 51 | # WPS326: "Found implicit string concatenation" -- nothing bad about this 52 | WPS326, 53 | # WPS422: "Found future import: *" -- we need these for multipython 54 | WPS422, 55 | 56 | # FIXME: These `flake8-annotations` errors need fixing and removal 57 | # ANN001: Missing type annotation for function argument 'argv' 58 | ANN001, 59 | # ANN002: Missing type annotation for *exceptions 60 | ANN002, 61 | # ANN003: Missing type annotation for **kwargs 62 | ANN003, 63 | # ANN101: Missing type annotation for self in method 64 | ANN101, 65 | # ANN102: Missing type annotation for cls in classmethod 66 | ANN102, 67 | # ANN201: Missing return type annotation for public function 68 | ANN201, 69 | # ANN202: Missing return type annotation for protected function 70 | ANN202, 71 | # ANN204: Missing return type annotation for special method 72 | ANN204, 73 | # ANN205: Missing return type annotation for staticmethod 74 | ANN205, 75 | # ANN206: Missing return type annotation for classmethod 76 | ANN206, 77 | 78 | # FIXME: These `flake8-spellcheck` errors need fixing and removal 79 | # SC100: Possibly misspelt word / comments 80 | SC100, 81 | # SC200: Possibly misspelt word / names 82 | SC200, 83 | 84 | # FIXME: Temporary WPS ignore 85 | WPS, 86 | # FIXME: DAR101 Missing parameter(s) in Docstring: - classifiers 87 | DAR101, 88 | # FIXME: DAR201 Missing "Returns" in Docstring: - return 89 | DAR201, 90 | # FIXME: DAR301 Missing "Yields" in Docstring: - yield 91 | DAR301, 92 | # FIXME: DAR401 Missing exception(s) in Raises section: -r LookupError 93 | DAR401, 94 | # FIXME: PT006: wrong name(s) type in @pytest.mark.parametrize, expected tuple 95 | # FIXME: PT022: no teardown in fixture tmp_git_repo, use return instead of yield 96 | PT006, 97 | PT022, 98 | # FIXME: LN002: doc/comment line is too long 99 | LN002, 100 | # FIXME: Temporary `flake8-logging` ignore 101 | G200, 102 | # FIXME: D401: First line should be in imperative mood; try rephrasing 103 | D401, 104 | 105 | 106 | format = default 107 | 108 | # Let's not overcomplicate the code: 109 | max-complexity = 10 110 | 111 | # Accessibility/large fonts and PEP8 friendly. 112 | # This is being flexibly extended through the `flake8-length`: 113 | max-line-length = 79 114 | 115 | # Allow certain violations in certain files: 116 | # Please keep both sections of this list sorted, as it will be easier for others to find and add entries in the future 117 | per-file-ignores = 118 | # The following ignores have been researched and should be considered permanent 119 | # each should be preceded with an explanation of each of the error codes 120 | # If other ignores are added for a specific file in the section following this, 121 | # these will need to be added to that line as well. 122 | 123 | # Sphinx builds aren't supposed to be run under Python 2: 124 | # docs/conf.py: WPS305 125 | 126 | # The package has imports exposing private things to the public: 127 | # src/pylibsshext/__init__.py: WPS412 128 | 129 | # There are multiple `assert`s (S101) 130 | # and subprocesses (import – S404; call – S603) in tests; 131 | # also, using fixtures looks like shadowing the outer scope (WPS442); 132 | # furthermore, we should be able to import and test private attributes 133 | # (WPS450) and modules (WPS436), and finally it's impossible to 134 | # have <= members in tests (WPS202), including many local vars (WPS210): 135 | tests/**.py: S101, S404, S603, WPS202, WPS210, WPS436, WPS442, WPS450 136 | 137 | 138 | # Count the number of occurrences of each error/warning code and print a report: 139 | statistics = true 140 | 141 | # ## Plugin-provided settings: ## 142 | 143 | # flake8-eradicate 144 | # E800: 145 | eradicate-whitelist-extend = isort:\s+\w+ 146 | 147 | # flake8-pytest-style 148 | # PT001: 149 | pytest-fixture-no-parentheses = true 150 | # PT006: 151 | pytest-parametrize-names-type = tuple 152 | # PT007: 153 | pytest-parametrize-values-type = tuple 154 | pytest-parametrize-values-row-type = tuple 155 | # PT023: 156 | pytest-mark-no-parentheses = true 157 | 158 | # flake8-rst-docstrings 159 | rst-directives = 160 | spelling 161 | rst-roles = 162 | # Built-in Sphinx roles: 163 | class, 164 | data, 165 | file, 166 | exc, 167 | meth, 168 | mod, 169 | term, 170 | py:class, 171 | py:data, 172 | py:exc, 173 | py:meth, 174 | py:term, 175 | # Sphinx's internal role: 176 | event, 177 | 178 | # wemake-python-styleguide 179 | show-source = true 180 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ci: 4 | autoupdate_schedule: quarterly # low frequency to reduce maintenance noise 5 | 6 | repos: 7 | 8 | 9 | - repo: https://github.com/asottile/add-trailing-comma.git 10 | rev: v3.1.0 11 | hooks: 12 | - id: add-trailing-comma 13 | 14 | - repo: https://github.com/PyCQA/isort.git 15 | rev: 5.13.2 16 | hooks: 17 | - id: isort 18 | args: 19 | - --honor-noqa 20 | 21 | - repo: https://github.com/Lucas-C/pre-commit-hooks.git 22 | rev: v1.5.5 23 | hooks: 24 | - id: remove-tabs 25 | 26 | - repo: https://github.com/python-jsonschema/check-jsonschema.git 27 | rev: 0.29.1 28 | hooks: 29 | - id: check-github-workflows 30 | files: ^\.github/workflows/[^/]+$ 31 | types: 32 | - yaml 33 | - id: check-jsonschema 34 | name: Check GitHub Workflows set timeout-minutes 35 | args: 36 | - --builtin-schema 37 | - github-workflows-require-timeout 38 | files: ^\.github/workflows/[^/]+$ 39 | types: 40 | - yaml 41 | - id: check-readthedocs 42 | 43 | - repo: https://github.com/Lucas-C/pre-commit-hooks-markup.git 44 | rev: v1.0.1 45 | hooks: 46 | - id: rst-linter 47 | files: README.rst 48 | 49 | - repo: https://github.com/pre-commit/pre-commit-hooks.git 50 | rev: v4.6.0 51 | hooks: 52 | # Side-effects: 53 | - id: trailing-whitespace 54 | - id: check-merge-conflict 55 | - id: double-quote-string-fixer 56 | - id: end-of-file-fixer 57 | - id: requirements-txt-fixer 58 | 59 | # Non-modifying checks: 60 | - id: name-tests-test 61 | - id: check-added-large-files 62 | - id: check-byte-order-marker 63 | - id: check-case-conflict 64 | # disabled due to pre-commit/pre-commit-hooks#159 65 | # - id: check-docstring-first 66 | - id: check-json 67 | - id: check-symlinks 68 | - id: check-yaml 69 | - id: detect-private-key 70 | 71 | # Heavy checks: 72 | - id: check-ast 73 | - id: debug-statements 74 | 75 | - repo: https://github.com/PyCQA/pydocstyle.git 76 | rev: 6.3.0 77 | hooks: 78 | - id: pydocstyle 79 | 80 | - repo: https://github.com/codespell-project/codespell.git 81 | rev: v2.3.0 82 | hooks: 83 | - id: codespell 84 | 85 | - repo: https://github.com/adrienverge/yamllint.git 86 | rev: v1.35.1 87 | hooks: 88 | - id: yamllint 89 | types: 90 | - file 91 | - yaml 92 | args: 93 | - --strict 94 | 95 | - repo: https://github.com/PyCQA/flake8.git 96 | rev: 7.1.1 97 | hooks: 98 | - id: flake8 99 | alias: flake8-no-wps 100 | name: flake8 WPS-excluded 101 | additional_dependencies: 102 | - darglint ~= 1.8.1 103 | - flake8-2020 ~= 1.7.0 104 | - flake8-annotations ~= 2.9.1; python_version >= "3.7" 105 | - flake8-annotations ~= 2.7.0; python_version < "3.7" 106 | - flake8-docstrings ~= 1.6.0 107 | - flake8-length ~= 0.3.0 108 | - flake8-logging-format ~= 0.7.5 109 | - flake8-pytest-style ~= 1.6.0 110 | - flake8-spellcheck ~= 0.28.0; python_version >= "3.8" 111 | - flake8-spellcheck ~= 0.26.0; python_version < "3.8" 112 | language_version: python3 113 | 114 | - repo: https://github.com/PyCQA/flake8.git 115 | rev: 7.1.1 116 | hooks: 117 | - id: flake8 118 | alias: flake8-only-wps 119 | name: flake8 WPS-only 120 | args: 121 | - --select 122 | - WPS 123 | additional_dependencies: 124 | - wemake-python-styleguide ~= 0.19.2 125 | language_version: python3.11 # flake8-commas doesn't work w/ Python 3.12 126 | 127 | - repo: https://github.com/pre-commit/mirrors-mypy.git 128 | rev: v1.11.2 129 | hooks: 130 | - id: mypy 131 | alias: mypy-py313 132 | name: MyPy, for Python 3.13 133 | additional_dependencies: 134 | - . 135 | - attrs 136 | - types-PyYAML 137 | args: 138 | - --namespace-packages 139 | - --pretty 140 | - --python-version=3.12 141 | - --show-column-numbers 142 | - --show-error-codes 143 | - --show-error-context 144 | - --strict-optional 145 | - -p 146 | - octomachinery 147 | - -p 148 | - tests 149 | pass_filenames: false 150 | - id: mypy 151 | alias: mypy-py311 152 | name: MyPy, for Python 3.11 153 | additional_dependencies: 154 | - . 155 | - attrs 156 | - types-PyYAML 157 | args: 158 | - --namespace-packages 159 | - --pretty 160 | - --python-version=3.11 161 | - --show-column-numbers 162 | - --show-error-codes 163 | - --show-error-context 164 | - --strict-optional 165 | - -p 166 | - octomachinery 167 | - -p 168 | - tests 169 | pass_filenames: false 170 | - id: mypy 171 | alias: mypy-py39 172 | name: MyPy, for Python 3.9 173 | additional_dependencies: 174 | - . 175 | - attrs 176 | - types-PyYAML 177 | args: 178 | - --namespace-packages 179 | - --pretty 180 | - --python-version=3.9 181 | - --show-column-numbers 182 | - --show-error-codes 183 | - --show-error-context 184 | - --strict-optional 185 | - -p 186 | - octomachinery 187 | - -p 188 | - tests 189 | pass_filenames: false 190 | - id: mypy 191 | alias: mypy-py37 192 | name: MyPy, for Python 3.7 193 | additional_dependencies: 194 | - . 195 | - attrs 196 | - types-PyYAML 197 | args: 198 | - --namespace-packages 199 | - --pretty 200 | - --python-version=3.8 201 | - --show-column-numbers 202 | - --show-error-codes 203 | - --show-error-context 204 | - --strict-optional 205 | - -p 206 | - octomachinery 207 | - -p 208 | - tests 209 | pass_filenames: false 210 | 211 | - repo: https://github.com/PyCQA/pylint.git 212 | rev: v3.2.6 213 | hooks: 214 | - id: pylint 215 | additional_dependencies: 216 | # runtime deps: 217 | - aiohttp 218 | - anyio < 2.0.0 219 | - click 220 | - cryptography 221 | - environ-config >= 19.1.0 222 | - envparse 223 | - gidgethub >= 4.2.0 224 | - pyjwt[crypto] 225 | - pyyaml 226 | - sentry_sdk 227 | - setuptools_scm 228 | # test deps: 229 | - pylint-pytest ~= 2.0.0a0 230 | - pytest 231 | - pytest-aiohttp 232 | - pytest-cov 233 | - pytest-xdist 234 | 235 | ... 236 | -------------------------------------------------------------------------------- /octomachinery/routing/webhooks_dispatcher.py: -------------------------------------------------------------------------------- 1 | """GitHub webhook events dispatching logic.""" 2 | 3 | from __future__ import annotations 4 | 5 | import contextlib 6 | import logging 7 | from typing import Any, Iterable 8 | 9 | from anyio import get_cancelled_exc_class 10 | from anyio import sleep as async_sleep 11 | 12 | import sentry_sdk 13 | 14 | # pylint: disable=relative-beyond-top-level,import-error 15 | from ..github.api.app_client import GitHubApp 16 | # pylint: disable=relative-beyond-top-level,import-error 17 | from ..github.entities.action import GitHubAction 18 | # pylint: disable=relative-beyond-top-level,import-error 19 | from ..github.errors import GitHubActionError 20 | # pylint: disable=relative-beyond-top-level,import-error 21 | from ..github.models.events import GitHubEvent 22 | # pylint: disable=relative-beyond-top-level,import-error 23 | from ..runtime.context import RUNTIME_CONTEXT 24 | 25 | 26 | __all__ = ('route_github_event',) 27 | 28 | 29 | logger = logging.getLogger(__name__) 30 | 31 | 32 | # pylint: disable=fixme 33 | async def route_github_event( # type: ignore[return] # FIXME 34 | *, 35 | github_event: GitHubEvent, 36 | github_app: GitHubApp, 37 | ) -> Iterable[Any]: 38 | """Dispatch GitHub event to corresponding handlers. 39 | 40 | Set up ``RUNTIME_CONTEXT`` before doing that. This is so 41 | the concrete event handlers have access to the API client 42 | and flags in runtime. 43 | """ 44 | is_gh_action = isinstance(github_app, GitHubAction) 45 | # pylint: disable=assigning-non-slot 46 | RUNTIME_CONTEXT.IS_GITHUB_ACTION = is_gh_action 47 | # pylint: disable=assigning-non-slot 48 | RUNTIME_CONTEXT.IS_GITHUB_APP = not is_gh_action 49 | 50 | # pylint: disable=assigning-non-slot 51 | RUNTIME_CONTEXT.github_app = github_app 52 | 53 | # pylint: disable=assigning-non-slot 54 | RUNTIME_CONTEXT.github_event = github_event 55 | 56 | # pylint: disable=assigning-non-slot 57 | RUNTIME_CONTEXT.app_installation = None 58 | if is_gh_action: 59 | # pylint: disable=assigning-non-slot 60 | RUNTIME_CONTEXT.app_installation_client = github_app.api_client 61 | else: 62 | with contextlib.suppress(LookupError): 63 | # pylint: disable=pointless-string-statement 64 | """Provision an installation API client if possible. 65 | 66 | Some events (like `ping`) are 67 | happening application/GitHub-wide and are not bound to 68 | a specific installation. The webhook payloads of such events 69 | don't contain any reference to an installation. 70 | Some events don't even refer to a GitHub App 71 | (e.g. `security_advisory`). 72 | """ 73 | github_install = await github_app.get_installation(github_event) 74 | # pylint: disable=assigning-non-slot 75 | RUNTIME_CONTEXT.app_installation = github_install 76 | # pylint: disable=assigning-non-slot 77 | RUNTIME_CONTEXT.app_installation_client = github_install.api_client 78 | 79 | # Give GitHub a sec to deal w/ eventual consistency. 80 | # This is only needed for events that arrive over HTTP. 81 | # If the dispatcher is invoked from GitHub Actions, 82 | # by the time it's invoked the action must be already consistently 83 | # distributed within GitHub's systems because spawning VMs takes time 84 | # and actions are executed in workflows that rely on those VMs. 85 | await async_sleep(1) 86 | 87 | try: 88 | return await github_app.dispatch_event(github_event) 89 | except GitHubActionError: 90 | # Bypass GitHub Actions errors as they are supposed to be a 91 | # mechanism for communicating outcomes and are expected. 92 | raise 93 | except get_cancelled_exc_class(): 94 | raise 95 | except Exception as exc: # pylint: disable=broad-except 96 | # NOTE: It's probably better to wrap each event handler with 97 | # NOTE: try/except and call `capture_exception()` there instead. 98 | # NOTE: We'll also need to figure out the magic of associating 99 | # NOTE: breadcrumbs with event handlers. 100 | sentry_sdk.capture_exception(exc) 101 | 102 | # NOTE: Framework-wise, these exceptions are meaningless because they 103 | # NOTE: can be anything random that the webhook author (octomachinery 104 | # NOTE: end-user) forgot to handle. There's nothing we can do about 105 | # NOTE: them except put in the log so that the end-user would be able 106 | # NOTE: to properly debug their problem by inspecting the logs. 107 | # NOTE: P.S. This is also where we'd inject Sentry 108 | if isinstance(exc.__context__, get_cancelled_exc_class()): 109 | # The CancelledError context is irrelevant to the 110 | # user-defined webhook event handler workflow so we're 111 | # dropping it from the logs: 112 | exc.__context__ = None 113 | 114 | logger.exception( 115 | 'An unhandled exception happened while running webhook ' 116 | 'event handlers for "%s"...', 117 | github_event.name, 118 | ) 119 | delivery_id_msg = ( 120 | '' if is_gh_action 121 | else ' (Delivery ID: ' 122 | # FIXME: # pylint: disable=fixme 123 | f'{github_event.delivery_id!s})' # type: ignore[attr-defined] 124 | ) 125 | logger.debug( 126 | 'The payload of "%s" event%s is: %r', 127 | github_event.name, delivery_id_msg, github_event.payload, 128 | ) 129 | 130 | if is_gh_action: 131 | # NOTE: In GitHub Actions env, the app is supposed to run as 132 | # NOTE: a foreground single event process rather than a 133 | # NOTE: server for multiple events. It's okay to propagate 134 | # NOTE: unhandled errors so that they are spit out to the 135 | # NOTE: console. 136 | raise 137 | except BaseException: # SystemExit + KeyboardInterrupt + GeneratorExit 138 | raise 139 | -------------------------------------------------------------------------------- /docs/howto-guides.rst: -------------------------------------------------------------------------------- 1 | How-to guides 2 | ============= 3 | 4 | 5 | Running a one-off task against GitHub API 6 | ----------------------------------------- 7 | 8 | Sometimes you need to run a series of queries against GitHub API. 9 | To do this, initialize a token (it's taken from the ``GITHUB_TOKEN`` env 10 | var in the example below), construct a GitHub API wrapper and you are 11 | good to go. 12 | 13 | :py:class:`~octomachinery.github.api.raw_client.RawGitHubAPI` is a 14 | wrapper around interface provided by 15 | :py:class:`~gidgethub.abc.GitHubAPI`, you can find the usage interface 16 | on its documentation page. Don't forget to specify a ``user_agent`` 17 | string — it's mandatory! 18 | 19 | API calls return native Python :py:class:`dict` or iterable objects. 20 | 21 | .. code:: python 22 | 23 | import asyncio 24 | import os 25 | 26 | from octomachinery.github.api.tokens import GitHubOAuthToken 27 | from octomachinery.github.api.raw_client import RawGitHubAPI 28 | 29 | 30 | async def main(): 31 | access_token = GitHubOAuthToken(os.environ["GITHUB_TOKEN"]) 32 | github_api = RawGitHubAPI(access_token, user_agent='webknjaz') 33 | await github_api.post( 34 | '/repos/mariatta/strange-relationship/issues', 35 | data={ 36 | 'title': 'We got a problem', 37 | 'body': 'Use more emoji!', 38 | }, 39 | ) 40 | 41 | 42 | asyncio.run(main()) 43 | 44 | 45 | Authenticating as a bot (GitHub App) 46 | ------------------------------------ 47 | 48 | To act as a bot, you should use a special kind of integration with 49 | GitHub — Apps. They are reusable entities in the GitHub Platform 50 | available to be installed into multiple accounts and organizations. 51 | 52 | Classic GitHub App requires a web-sever part to be deployed somewhere on 53 | the Internet in order to receive events GitHub Platform would send 54 | there. 55 | 56 | Yet, sometimes, you just want to act as a bot without any of that 57 | deployment hustle. You may want to have a ``[bot]`` label next to 58 | comments you'll post via API. You may want to manage rate limits better. 59 | This will allow you to run one-off tasks like batch changes/migrations. 60 | 61 | You'll still need to register a GitHub App and install it into the 62 | target user account or organization. You'll also have to specify APIs 63 | you'd like to access using this App. 64 | 65 | Then, you'll need to get your App's ID and a private key. 66 | 67 | Now, first, specify the App ID, the key path and the target account. 68 | After that, create a 69 | :py:class:`~octomachinery.github.config.app.GitHubAppIntegrationConfig` 70 | instance also specifying app name, version and some URL (these will be 71 | used to generate a ``User-Agent`` HTTP header for API queries). 72 | Then, create a 73 | :py:class:`~octomachinery.github.api.app_client.GitHubApp` instance from 74 | that config. Retrieve a list of places where the App is installed, 75 | filter out the target Installation and get an API client for it. 76 | Finally, use 77 | :py:class:`~octomachinery.github.api.raw_client.RawGitHubAPI` as usual. 78 | 79 | .. code:: python 80 | 81 | import asyncio 82 | import pathlib 83 | 84 | from aiohttp.client import ClientSession 85 | 86 | from octomachinery.github.api.app_client import GitHubApp 87 | from octomachinery.github.config.app import GitHubAppIntegrationConfig 88 | 89 | 90 | target_github_account_or_org = 'webknjaz' # where the app is installed to 91 | 92 | github_app_id = 12345 93 | github_app_private_key_path = pathlib.Path( 94 | '~/Downloads/star-wars.2011-05-04.private-key.pem', 95 | ).expanduser().resolve() 96 | 97 | github_app_config = GitHubAppIntegrationConfig( 98 | app_id=github_app_id, 99 | private_key=github_app_private_key_path.read_text(), 100 | 101 | app_name='MyGitHubClient', 102 | app_version='1.0', 103 | app_url='https://awesome-app.dev', 104 | ) 105 | 106 | 107 | async def get_github_client(github_app, account): 108 | github_app_installations = await github_app.get_installations() 109 | target_github_app_installation = next( # find the one 110 | ( 111 | i for n, i in github_app_installations.items() 112 | if i._metadata.account['login'] == account 113 | ), 114 | None, 115 | ) 116 | return target_github_app_installation.api_client 117 | 118 | 119 | async def main(): 120 | async with ClientSession() as http_session: 121 | github_app = GitHubApp(github_app_config, http_session) 122 | github_api = await get_github_client( 123 | github_app, target_github_account_or_org, 124 | ) 125 | user = await github_api.getitem( 126 | '/users/{account_name}', 127 | url_vars={'account_name': target_github_account_or_org}, 128 | ) 129 | print(f'User found: {user["login"]}') 130 | print(f'Rate limit stats: {github_api.rate_limit!s}') 131 | 132 | 133 | asyncio.run(main()) 134 | 135 | 136 | Making API queries against preview endpoints 137 | -------------------------------------------- 138 | 139 | Endpoints with stable interfaces in GitHub API are easy to hit. But some 140 | are marked as preview API. For those, GitHub requires special Accept 141 | headers to be passed along with a normal HTTP request. The exact strings 142 | are documented at https://developers.github.com under specific endpoint 143 | sections in their description. 144 | 145 | Given that you've already got an instance of 146 | :py:class:`~octomachinery.github.api.raw_client.RawGitHubAPI` 147 | initialized, what's left is to pass ``preview_api_version`` argument 148 | with the appropriate preview API code name when making query to the API 149 | endpoint requiring that. 150 | 151 | .. code:: python 152 | 153 | github_api: RawGitHubAPI 154 | 155 | repo_slug = 'sanitizers/octomachinery' 156 | issue_number = 15 157 | 158 | await github_api.post( 159 | f'/repos/{repo_slug}/issues/{issue_number}/reactions', 160 | preview_api_version='squirrel-girl', 161 | data={'content': 'heart'}, 162 | ) 163 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = python 3 | minversion = 3.21.0 4 | requires = 5 | setuptools >= 40.9.0 6 | pip >= 19.0.3 7 | # tox-venv >= 0.4.0 8 | isolated_build = true 9 | 10 | 11 | [testenv] 12 | package = wheel 13 | wheel_build_env = .pkg 14 | basepython = python3 15 | isolated_build = true 16 | usedevelop = false 17 | extras = 18 | testing 19 | commands = 20 | {envpython} -m \ 21 | pytest \ 22 | {tty:--color=yes} \ 23 | {posargs:--cov-report=term-missing:skip-covered} 24 | 25 | 26 | [testenv:old-deps] 27 | description = 28 | Run tests with the lowest possible dependency versions as opposed to 29 | using the latest ones 30 | commands_pre = 31 | # NOTE: Not using `deps = ` for this because installing the dist 32 | # NOTE: happens later and overrides it. 33 | # NOTE: With this step we're setting certain packages to the lowest 34 | # NOTE: versions — either limited by the runtime deps or just known 35 | # NOTE: to work. 36 | # NOTE: For example, PyJWT versions below 1.4.2 cause TypeError when 37 | # NOTE: the PEM key is passed as bytes. 38 | # Ref: https://github.com/sanitizers/octomachinery/issues/46: 39 | {envpython} -m pip install \ 40 | 'cryptography == 2.4.2' \ 41 | 'gidgethub == 4.2.0' \ 42 | 'pyjwt == 1.4.2; python_version < "3.10"' \ 43 | 'pyjwt == 1.7.0; python_version >= "3.10"' \ 44 | 'pytest-aiohttp == 0.1.0' \ 45 | 'setuptools == 35.0.2; python_version < "3.10"' \ 46 | 'setuptools == 41; python_version >= "3.10"' \ 47 | 'setuptools-scm == 1.15.4' 48 | setenv = 49 | PYTEST_ADDOPTS = -p no:warnings 50 | 51 | 52 | [testenv:check-docs] 53 | basepython = python3 54 | isolated_build = true 55 | # `usedevelop = true` overrides `skip_install` instruction, it's unwanted 56 | usedevelop = false 57 | ## don't install octomachinery itself in this env 58 | #skip_install = true 59 | #usedevelop = true 60 | extras = 61 | docs 62 | # testing 63 | #deps = 64 | # pip >= 18 65 | changedir = docs 66 | commands = 67 | {envpython} -m sphinx \ 68 | -b linkcheck \ 69 | -j auto \ 70 | {tty:--color} \ 71 | -n \ 72 | -d {toxinidir}/build/.doctrees \ 73 | . \ 74 | {toxinidir}/build/html 75 | 76 | 77 | [testenv:build-docs] 78 | basepython = python3 79 | isolated_build = true 80 | # `usedevelop = true` overrides `skip_install` instruction, it's unwanted 81 | usedevelop = false 82 | ## don't install octomachinery itself in this env 83 | #skip_install = true 84 | #usedevelop = true 85 | extras = 86 | docs 87 | # testing 88 | #deps = 89 | # pip >= 18 90 | changedir = docs 91 | commands = 92 | # FIXME: Add -W option below once all other warnings are gone 93 | {envpython} -m sphinx \ 94 | -j auto \ 95 | -b html \ 96 | {tty:--color} \ 97 | -n \ 98 | -d "{toxinidir}/build/.doctrees" \ 99 | . \ 100 | "{toxinidir}/build/html" 101 | 102 | # Print out the output docs dir and a way to serve html: 103 | -{envpython} -c \ 104 | 'import pathlib; docs_dir = pathlib.Path(r"{toxinidir}") / "build" / "html"; index_file = docs_dir / "index.html"; '\ 105 | 'print("\n" + "=" * 120 + f"\n\nDocumentation available under `file://\{index_file\}`\n\nTo serve docs, use `python3 -m http.server --directory \{docs_dir\} 0`\n\n" + "=" * 120)' 106 | 107 | 108 | [testenv:build-dists] 109 | description = 110 | Build dists and put them into the dist{/} folder 111 | basepython = python3 112 | isolated_build = true 113 | # `usedevelop = true` overrides `skip_install` instruction, it's unwanted 114 | usedevelop = false 115 | # don't install octomachinery itself in this env 116 | skip_install = true 117 | deps = 118 | build >= 0.3.1, < 0.4.0 119 | commands = 120 | {envpython} -c \ 121 | "import shutil; \ 122 | shutil.rmtree('{toxinidir}{/}dist{/}', ignore_errors=True)" 123 | 124 | {envpython} -m build \ 125 | --outdir '{toxinidir}{/}dist{/}' \ 126 | {posargs:--sdist --wheel} \ 127 | '{toxinidir}' 128 | 129 | 130 | [testenv:metadata-validation] 131 | depends = 132 | build-dists 133 | deps = 134 | twine 135 | description = 136 | Verify that dists under the `dist{/}` 137 | dir have valid metadata 138 | # Ref: https://twitter.com/di_codes/status/1044358639081975813 139 | commands = 140 | {envpython} -m \ 141 | twine check \ 142 | {toxinidir}{/}dist{/}* 143 | 144 | # Install an sdist and a wheel into tmp dirs for further comparison 145 | {envpython} -c \ 146 | "import shutil; \ 147 | shutil.rmtree('{temp_dir}{/}.installed{/}', ignore_errors=True)" 148 | {envpython} -m \ 149 | pip install octomachinery \ 150 | --no-index \ 151 | -f "{toxinidir}{/}dist{/}" \ 152 | --no-deps \ 153 | --only-binary octomachinery \ 154 | --no-compile --no-cache-dir \ 155 | -t "{temp_dir}{/}.installed{/}from-whl{/}" 156 | # Pre-download build deps for installing sdist 157 | {envpython} -m \ 158 | pip download \ 159 | setuptools \ 160 | setuptools_scm \ 161 | setuptools_scm_git_archive \ 162 | wheel \ 163 | -d "{temp_dir}{/}.build-deps{/}" 164 | {envpython} -m \ 165 | pip install octomachinery \ 166 | --no-index \ 167 | -f "{toxinidir}{/}dist{/}" \ 168 | -f "{temp_dir}{/}.build-deps{/}" \ 169 | --no-deps \ 170 | --no-binary octomachinery \ 171 | --no-compile --no-cache-dir \ 172 | -t "{temp_dir}{/}.installed{/}from-sdist{/}" 173 | 174 | # Normalize known content difference 175 | sh -c 'sed -i "s#^\(octomachinery-.*.dist-info/top_level.txt,sha256=\).*#\1#" \ 176 | {temp_dir}{/}.installed{/}from-*{/}octomachinery-*.dist-info{/}RECORD' 177 | sh -c 'rm -rfv \ 178 | {temp_dir}{/}.installed{/}from-*{/}octomachinery-*.dist-info{/}top_level.txt' 179 | 180 | # Compare wheel and sdist installs recursively as a smoke-test 181 | diff -ur \ 182 | "{temp_dir}{/}.installed{/}from-whl" \ 183 | "{temp_dir}{/}.installed{/}from-sdist" 184 | skip_install = true 185 | usedevelop = false 186 | allowlist_externals = 187 | diff 188 | sh 189 | 190 | 191 | [testenv:pre-commit] 192 | isolated_build = true 193 | deps = 194 | pre-commit 195 | commands = 196 | {envpython} -m pre_commit run --show-diff-on-failure {posargs:--all-files} 197 | 198 | # Print out the advise of how to install pre-commit from this env into Git: 199 | -{envpython} -c \ 200 | 'cmd = "{envpython} -m pre_commit install"; scr_width = len(cmd) + 10; sep = "=" * scr_width; cmd_str = " $ " + cmd; '\ 201 | 'print("\n" + sep + "\nTo install pre-commit hooks into the Git repo, run:\n\n" + cmd_str + "\n\n" + sep + "\n")' 202 | skip_install = true 203 | -------------------------------------------------------------------------------- /octomachinery/cli/__main__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | """Octomachinery CLI entrypoint.""" 3 | 4 | import asyncio 5 | import importlib 6 | import os 7 | import pathlib 8 | from functools import wraps 9 | from io import TextIOWrapper 10 | from typing import ( 11 | Any, Callable, Dict, FrozenSet, Iterable, Iterator, Set, Union, 12 | ) 13 | 14 | from aiohttp.client import ClientSession 15 | 16 | import click 17 | 18 | # pylint: disable=relative-beyond-top-level 19 | from ..app.config import BotAppConfig 20 | # pylint: disable=relative-beyond-top-level 21 | from ..app.routing.abc import OctomachineryRouterBase 22 | # pylint: disable=relative-beyond-top-level 23 | from ..app.routing.webhooks_dispatcher import route_github_event 24 | # pylint: disable=relative-beyond-top-level 25 | from ..github.api.app_client import GitHubApp 26 | # pylint: disable=relative-beyond-top-level 27 | from ..github.entities.action import GitHubAction 28 | # pylint: disable=relative-beyond-top-level 29 | # pylint: disable=relative-beyond-top-level 30 | from ..github.models.events import GitHubEvent, GitHubWebhookEvent 31 | 32 | 33 | @click.group() 34 | @click.pass_context 35 | def cli(ctx: click.Context) -> None: # pylint: disable=unused-argument 36 | """Click CLI base.""" 37 | 38 | 39 | def run_async(orig_async_func: Callable[..., Any]): 40 | """Run the given async func in event loop.""" 41 | @wraps(orig_async_func) 42 | def func_wrapper(*args: Any, **kwargs: Any): 43 | return asyncio.run(orig_async_func(*args, **kwargs)) 44 | return func_wrapper 45 | 46 | 47 | @cli.command() 48 | @click.option('--event', '-e', prompt=False, type=str) 49 | @click.option( 50 | '--payload-path', '-p', 51 | 'event_payload', 52 | prompt=True, 53 | type=click.File(mode='r'), 54 | ) 55 | @click.option('--token', '-t', prompt=False, type=str) 56 | @click.option('--app', '-a', prompt=False, type=int) 57 | @click.option('--private-key', '-P', prompt=False, type=click.File(mode='r')) 58 | @click.option('--entrypoint-module', '-m', prompt=False, type=str) 59 | @click.option( 60 | '--event-router', '-r', 61 | 'event_routers', 62 | multiple=True, 63 | prompt=False, 64 | type=str, 65 | ) 66 | @click.pass_context 67 | @run_async 68 | async def receive( # pylint: disable=too-many-arguments,too-many-locals 69 | ctx: click.Context, 70 | event: str, event_payload: TextIOWrapper, 71 | token: str, 72 | app: int, private_key: TextIOWrapper, 73 | entrypoint_module: str, 74 | event_routers: Iterable[str], 75 | ) -> None: 76 | """Webhook event receive command.""" 77 | app_missing_private_key = app is not None and not private_key 78 | if app_missing_private_key: 79 | ctx.fail(click.style('App requires a private key', fg='red')) 80 | 81 | creds_present = token or (app and private_key) 82 | if not creds_present: 83 | ctx.fail(click.style('GitHub auth credentials are missing', fg='red')) 84 | 85 | too_many_creds_present = token and (app or private_key) 86 | if too_many_creds_present: 87 | ctx.fail( 88 | click.style( 89 | 'Please choose between a token or ' 90 | 'an app id with a private key', 91 | fg='red', 92 | ), 93 | ) 94 | 95 | make_event = GitHubEvent if app is None else GitHubWebhookEvent 96 | try: 97 | gh_event = make_event.from_fixture_fd(event_payload, event=event) 98 | except ValueError as val_err: 99 | ctx.fail(click.style(str(val_err), fg='red')) 100 | 101 | os.environ.update(get_extra_env_vars(gh_event, token, app, private_key)) 102 | 103 | try: 104 | target_routers = set( 105 | load_event_routers( 106 | entrypoint_module, 107 | # pylint: disable=fixme 108 | event_routers, # type: ignore[arg-type] # FIXME: typing 109 | ), 110 | ) 111 | except AttributeError as attr_err: 112 | ctx.fail( 113 | click.style( 114 | f'Could not find an event router: {attr_err!s}', 115 | fg='red', 116 | ), 117 | ) 118 | except ImportError as imp_err: 119 | ctx.fail( 120 | click.style(f'Could not load a module: {imp_err!s}', fg='red'), 121 | ) 122 | 123 | config = BotAppConfig.from_dotenv() 124 | gh_app_kwargs = {'config': config.github} 125 | make_gh_app = GitHubApp 126 | if app is None: 127 | make_gh_app = GitHubAction 128 | gh_app_kwargs['metadata'] = config.action 129 | async with ClientSession() as http_client_session: 130 | github_app = make_gh_app( 131 | http_session=http_client_session, 132 | # FIXME: typing # pylint: disable=fixme 133 | event_routers=target_routers or None, # type: ignore[arg-type] 134 | **gh_app_kwargs, 135 | ) 136 | await route_github_event( 137 | github_event=gh_event, github_app=github_app, 138 | ) 139 | 140 | click.echo( 141 | click.style( 142 | f'Finished processing {gh_event.name!s} event!', 143 | fg='green', 144 | ), 145 | ) 146 | 147 | 148 | def load_event_routers( 149 | entrypoint_module: Union[str, None] = None, 150 | event_routers: Union[FrozenSet[str], Set[str]] = frozenset(), 151 | ) -> Iterator[OctomachineryRouterBase]: 152 | """Yield event routers from strings.""" 153 | if entrypoint_module is not None: 154 | importlib.import_module(entrypoint_module) 155 | 156 | for router_path in event_routers: 157 | target_sep = ':' if ':' in router_path else '.' 158 | module_path, _sep, target_router = router_path.rpartition(target_sep) 159 | yield getattr(importlib.import_module(module_path), target_router) 160 | 161 | 162 | def get_extra_env_vars( 163 | gh_event: GitHubEvent, token: str, app: int, 164 | private_key: TextIOWrapper, 165 | ) -> Dict[str, str]: 166 | """Construct additional env vars for App or Action processing.""" 167 | env = {} 168 | 169 | if app is not None: 170 | env['OCTOMACHINERY_APP_MODE'] = 'app' 171 | 172 | env['GITHUB_APP_IDENTIFIER'] = str(app) 173 | env['GITHUB_PRIVATE_KEY'] = private_key.read() 174 | return env 175 | 176 | env['OCTOMACHINERY_APP_MODE'] = 'action' 177 | 178 | env['GITHUB_ACTION'] = 'Fake CLI Action' 179 | env['GITHUB_ACTOR'] = gh_event.payload.get('sender', {}).get('login', '') 180 | env['GITHUB_EVENT_NAME'] = gh_event.name 181 | env['GITHUB_WORKSPACE'] = str(pathlib.Path('.').resolve()) 182 | env['GITHUB_SHA'] = gh_event.payload.get('head_commit', {}).get('id', '') 183 | env['GITHUB_REF'] = gh_event.payload.get('ref', '') 184 | env['GITHUB_REPOSITORY'] = ( 185 | gh_event.payload. 186 | get('repository', {}). 187 | get('full_name', '') 188 | ) 189 | env['GITHUB_TOKEN'] = token 190 | env['GITHUB_WORKFLOW'] = 'Fake CLI Workflow' 191 | env['GITHUB_EVENT_PATH'] = '/dev/null' 192 | 193 | return env 194 | 195 | 196 | def main(): 197 | """CLI entrypoint.""" 198 | kwargs = { 199 | 'prog_name': f'python3 -m {__package__}', 200 | } if __name__ == '__main__' else {} 201 | 202 | return cli( # pylint: disable=unexpected-keyword-arg 203 | auto_envvar_prefix='OCTOMACHINERY_CLI_', 204 | obj={}, 205 | **kwargs, 206 | ) 207 | 208 | 209 | __name__ == '__main__' and main() # pylint: disable=expression-not-assigned 210 | -------------------------------------------------------------------------------- /octomachinery/github/models/checks_api_requests.py: -------------------------------------------------------------------------------- 1 | """Models representing objects in GitHub Checks API.""" 2 | 3 | from functools import partial 4 | from typing import List, Optional 5 | 6 | import attr 7 | 8 | 9 | __all__ = ('NewCheckRequest', 'UpdateCheckRequest', 'to_gh_query') 10 | 11 | 12 | str_attrib = partial( # pylint: disable=invalid-name 13 | attr.ib, 14 | converter=lambda s: str(s) if s is not None else '', 15 | ) 16 | 17 | int_attrib = partial(attr.ib, converter=int) # pylint: disable=invalid-name 18 | 19 | optional_attrib = partial( # pylint: disable=invalid-name 20 | attr.ib, 21 | default=None, 22 | ) 23 | 24 | optional_int_attrib = partial( # pylint: disable=invalid-name 25 | optional_attrib, 26 | validator=attr.validators.optional(lambda *_: int(_[-1])), 27 | ) 28 | 29 | optional_str_attrib = partial( # pylint: disable=invalid-name 30 | optional_attrib, 31 | validator=attr.validators.optional(lambda *_: str(_[-1])), 32 | ) 33 | 34 | optional_list_attrib = partial( # pylint: disable=invalid-name 35 | attr.ib, 36 | default=[], 37 | validator=attr.validators.optional(lambda *_: list(_[-1])), 38 | ) 39 | 40 | 41 | def optional_converter(kwargs_dict, convert_to_cls): 42 | """Instantiate a class instances from dict.""" 43 | if kwargs_dict is not None and not isinstance(kwargs_dict, convert_to_cls): 44 | return convert_to_cls(**kwargs_dict) 45 | return kwargs_dict 46 | 47 | 48 | def optional_list_converter(args_list, convert_to_cls): 49 | """Convert list items to class instances.""" 50 | if args_list is not None and isinstance(args_list, list): 51 | return [ 52 | optional_converter(kwargs_dict, convert_to_cls) 53 | for kwargs_dict in args_list 54 | ] 55 | return args_list 56 | 57 | 58 | @attr.dataclass 59 | class CheckAnnotation: # pylint: disable=too-few-public-methods 60 | """Checks API annotation struct.""" 61 | 62 | path: str = str_attrib() 63 | start_line: int = int_attrib() 64 | end_line: int = int_attrib() 65 | annotation_level: str = str_attrib( 66 | validator=attr.validators.in_( 67 | ( 68 | 'notice', 69 | 'warning', 70 | 'failure', 71 | ), 72 | ), 73 | ) 74 | message: str = str_attrib() 75 | start_column: Optional[int] = optional_int_attrib() 76 | end_column: Optional[int] = optional_int_attrib() 77 | title: Optional[str] = optional_str_attrib() 78 | raw_details: Optional[str] = optional_str_attrib() 79 | 80 | 81 | @attr.dataclass 82 | class CheckImage: # pylint: disable=too-few-public-methods 83 | """Checks API image struct.""" 84 | 85 | alt: str = str_attrib() 86 | image_url: str = str_attrib() 87 | caption: Optional[str] = optional_str_attrib() 88 | 89 | 90 | @attr.dataclass 91 | class CheckActions: 92 | """Checks API actions struct.""" 93 | 94 | label: str = str_attrib() 95 | description: str = str_attrib() 96 | identifier: str = str_attrib() 97 | 98 | @label.validator 99 | def label_up_to_20(self, attribute, value): 100 | """Ensure that label is under 20.""" 101 | if len(value) > 20: 102 | raise ValueError( 103 | f'`{attribute.name}` must not exceed 20 characters.', 104 | ) 105 | 106 | @description.validator 107 | def description_up_to_40(self, attribute, value): 108 | """Ensure that description is under 40.""" 109 | if len(value) > 40: 110 | raise ValueError( 111 | f'`{attribute.name}` must not exceed 40 characters.', 112 | ) 113 | 114 | @identifier.validator 115 | def identifier_up_to_20(self, attribute, value): 116 | """Ensure that identifier is under 20.""" 117 | if len(value) > 20: 118 | raise ValueError( 119 | f'`{attribute.name}` must not exceed 20 characters.', 120 | ) 121 | 122 | 123 | @attr.dataclass 124 | class CheckOutput: # pylint: disable=too-few-public-methods 125 | """Checks API output struct.""" 126 | 127 | title: str = str_attrib() 128 | summary: str = str_attrib() 129 | text: str = str_attrib(default='') 130 | annotations: List[CheckAnnotation] = optional_list_attrib( 131 | converter=partial( 132 | optional_list_converter, 133 | convert_to_cls=CheckAnnotation, 134 | ), 135 | ) 136 | images: List[CheckImage] = optional_list_attrib( 137 | converter=partial(optional_list_converter, convert_to_cls=CheckImage), 138 | ) 139 | 140 | 141 | @attr.dataclass 142 | class BaseCheckRequestMixin: 143 | """Checks API base check request mixin.""" 144 | 145 | name: str = str_attrib() 146 | details_url: Optional[str] = optional_str_attrib() 147 | external_id: Optional[str] = optional_str_attrib() 148 | status: Optional[str] = attr.ib( 149 | default='queued', 150 | validator=attr.validators.optional( # type: ignore[arg-type] 151 | attr.validators.in_( 152 | ( 153 | 'queued', 154 | 'in_progress', 155 | 'completed', 156 | ), 157 | ), 158 | ), 159 | ) 160 | # '2018-05-27T14:30:33Z', datetime.isoformat(): 161 | started_at: Optional[str] = optional_str_attrib() 162 | conclusion: Optional[str] = attr.ib( 163 | # [required] if 'status' is set to 'completed', 164 | # should be missing if it's unset 165 | default=None, 166 | validator=attr.validators.optional( 167 | attr.validators.in_( 168 | ( 169 | 'success', 170 | 'failure', 171 | 'neutral', 172 | 'cancelled', 173 | 'timed_out', 174 | 'action_required', 175 | ), 176 | ), 177 | ), 178 | ) 179 | # [required] if 'conclusion' is set # '2018-05-27T14:30:33Z': 180 | completed_at: Optional[str] = optional_str_attrib() 181 | output: Optional[CheckOutput] = optional_attrib( 182 | converter=partial(optional_converter, convert_to_cls=CheckOutput), 183 | ) 184 | actions: List[CheckActions] = optional_list_attrib( 185 | converter=partial( 186 | optional_list_converter, 187 | convert_to_cls=CheckActions, 188 | ), 189 | ) 190 | 191 | @conclusion.validator 192 | def depends_on_status(self, attribute, value): 193 | """Ensure that conclusion is present if there's status.""" 194 | if self.status == 'completed' and not value: 195 | raise ValueError( 196 | f'`{attribute.name}` must be provided if status is completed', 197 | ) 198 | 199 | @completed_at.validator 200 | def depends_on_conclusion(self, attribute, value): 201 | """Ensure that completed is present if there's conclusion.""" 202 | if self.conclusion is not None and not value: 203 | raise ValueError( 204 | f'`{attribute.name}` must be provided ' 205 | 'if conclusion is present', 206 | ) 207 | 208 | @actions.validator 209 | def actions_up_to_3(self, attribute, value): 210 | """Ensure that the number of actions is below 3.""" 211 | if value is not None and len(value) > 3: 212 | raise ValueError(f'`{attribute.name}` must not exceed 3 items.') 213 | 214 | 215 | @attr.dataclass 216 | class NewCheckRequestMixin: # pylint: disable=too-few-public-methods 217 | """Checks API new check request mixin.""" 218 | 219 | head_branch: str = str_attrib() 220 | head_sha: str = str_attrib() 221 | 222 | 223 | @attr.dataclass 224 | class NewCheckRequest(NewCheckRequestMixin, BaseCheckRequestMixin): 225 | """Checks API new check request.""" 226 | 227 | 228 | @attr.dataclass 229 | class UpdateCheckRequest(BaseCheckRequestMixin): 230 | """Checks API update check request.""" 231 | 232 | 233 | def conditional_to_gh_query(req): 234 | """Traverse Checks API request structure.""" 235 | if hasattr(req, '__attrs_attrs__'): 236 | return to_gh_query(req) 237 | if isinstance(req, list): 238 | return list(map(conditional_to_gh_query, req)) 239 | if isinstance(req, dict): 240 | return { 241 | k: conditional_to_gh_query(v) for k, v in req.items() 242 | if v is not None or (isinstance(v, (list, dict)) and not v) 243 | } 244 | return req 245 | 246 | 247 | def to_gh_query(req): 248 | """Convert Checks API request object into a dict.""" 249 | return { 250 | k: conditional_to_gh_query(v) # recursive if dataclass or list 251 | for k, v in attr.asdict(req).items() 252 | if v is not None or (isinstance(v, (list, dict)) and not v) 253 | } 254 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Configuration of Sphinx documentation generator. 3 | 4 | This file does only contain a selection of the most common options. For a 5 | full list see the documentation: 6 | http://www.sphinx-doc.org/en/master/config 7 | """ 8 | 9 | from email import message_from_string 10 | from itertools import chain 11 | 12 | import pkg_resources 13 | 14 | 15 | # -- Path setup -------------------------------------------------------------- 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | # 21 | # import os 22 | # import sys 23 | # sys.path.insert(0, os.path.abspath('.')) 24 | 25 | 26 | # -- Project information ----------------------------------------------------- 27 | 28 | def get_supported_pythons(classifiers): 29 | """Return min and max supported Python version from meta as tuples.""" 30 | py_ver_classifier = 'Programming Language :: Python :: ' 31 | vers = filter(lambda c: c.startswith(py_ver_classifier), classifiers) 32 | vers = map(lambda c: c[len(py_ver_classifier):], vers) 33 | vers = filter(lambda c: c[0].isdigit() and '.' in c, vers) 34 | vers = map(lambda c: tuple(c.split('.')), vers) 35 | vers = sorted(vers) 36 | del vers[1:-1] 37 | if len(vers) < 2: 38 | vers *= 2 39 | return vers 40 | 41 | 42 | def get_github_data(project_urls): 43 | """Retrieve GitHub user/org and repo name from a bunch of links.""" 44 | partitioned_urls = (p.partition(', ') for p in project_urls) 45 | for _url_type, _sep, url in partitioned_urls: 46 | proto, _gh, uri = url.partition('://github.com/') 47 | if proto not in ('http', 'https'): 48 | continue 49 | return uri.split('/')[:2] 50 | 51 | raise LookupError('There are no project URLs pointing to GitHub') 52 | 53 | 54 | PYTHON_DISTRIBUTION_NAME = 'octomachinery' 55 | 56 | PRJ_DIST = pkg_resources.get_distribution(PYTHON_DISTRIBUTION_NAME) 57 | PRJ_PKG_INFO = PRJ_DIST.get_metadata(PRJ_DIST.PKG_INFO) 58 | PRJ_META = message_from_string(PRJ_PKG_INFO) 59 | PRJ_AUTHOR = PRJ_META['Author'] 60 | PRJ_LICENSE = PRJ_META['License'] 61 | PRJ_SUMMARY = PRJ_META['Summary'] 62 | PRJ_DESCRIPTION = PRJ_META['Description'] 63 | PRJ_PY_VER_RANGE = get_supported_pythons(PRJ_META.get_all('Classifier')) 64 | PRJ_PY_MIN_SUPPORTED, PRJ_PY_MAX_SUPPORTED = map('.'.join, PRJ_PY_VER_RANGE) 65 | PRJ_GITHUB_USER, PRJ_GITHUB_REPO = get_github_data( 66 | chain( 67 | (PRJ_META['Home-page'],), 68 | PRJ_META.get_all('Project-URL'), 69 | ), 70 | ) 71 | 72 | project = PRJ_DIST.project_name # pylint: disable=invalid-name 73 | author = PRJ_AUTHOR # pylint: disable=invalid-name 74 | copyright = f'2019, {author}' # pylint: disable=invalid-name,redefined-builtin 75 | 76 | # The full version, including alpha/beta/rc tags 77 | release = PRJ_DIST.version # pylint: disable=invalid-name 78 | # The short X.Y version 79 | # pylint: disable=invalid-name 80 | version = pkg_resources.parse_version(release).base_version 81 | 82 | rst_epilog = f""" 83 | .. |project| replace:: {project} 84 | .. |min_py_supported| replace:: {PRJ_PY_MIN_SUPPORTED} 85 | .. |max_py_supported| replace:: {PRJ_PY_MAX_SUPPORTED} 86 | """ # pylint: disable=invalid-name 87 | 88 | 89 | # -- General configuration --------------------------------------------------- 90 | 91 | # If your documentation needs a minimal Sphinx version, state it here. 92 | # 93 | needs_sphinx = '1.7.5' 94 | 95 | # Add any Sphinx extension module names here, as strings. They can be 96 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 97 | # ones. 98 | extensions = [ 99 | 'sphinx.ext.autodoc', 100 | 'sphinx.ext.doctest', 101 | 'sphinx.ext.intersphinx', 102 | 'sphinx.ext.todo', 103 | 'sphinx.ext.coverage', 104 | 'sphinx.ext.mathjax', 105 | 'sphinx.ext.ifconfig', 106 | 'sphinx.ext.viewcode', 107 | 'sphinx.ext.githubpages', 108 | 'sphinxcontrib.apidoc', 109 | ] 110 | 111 | # sphinxcontrib.apidoc configuration options 112 | apidoc_extra_args = ['--implicit-namespaces', '../octomachinery'] 113 | apidoc_module_dir = '.' 114 | apidoc_output_dir = 'reference' 115 | apidoc_separate_modules = True 116 | 117 | # Add any paths that contain templates here, relative to this directory. 118 | templates_path = ['_templates'] 119 | 120 | # The suffix(es) of source filenames. 121 | # You can specify multiple suffix as a list of string: 122 | # 123 | # source_suffix = ['.rst', '.md'] 124 | source_suffix = '.rst' 125 | 126 | # The master toctree document. 127 | master_doc = 'index' 128 | 129 | # The language for content autogenerated by Sphinx. Refer to documentation 130 | # for a list of supported languages. 131 | # 132 | # This is also used if you do content translation via gettext catalogs. 133 | # Usually you set "language" from the command line for these cases. 134 | language = None 135 | 136 | # List of patterns, relative to source directory, that match files and 137 | # directories to ignore when looking for source files. 138 | # This pattern also affects html_static_path and html_extra_path . 139 | exclude_patterns = [] 140 | 141 | # The name of the Pygments (syntax highlighting) style to use. 142 | # NOTE: These values are commented out so that Furo fall back to a nice style 143 | # pygments_style = 'sphinx' 144 | # pygments_dark_style = 'monokai' 145 | 146 | 147 | # -- Options for HTML output ------------------------------------------------- 148 | 149 | # The theme to use for HTML and HTML Help pages. See the documentation for 150 | # a list of builtin themes. 151 | # 152 | html_theme = 'furo' 153 | 154 | # Theme options are theme-specific and customize the look and feel of a theme 155 | # further. For a list of options available for each theme, see the 156 | # documentation. 157 | # 158 | # html_theme_options = {} 159 | 160 | # Add any paths that contain custom static files (such as style sheets) here, 161 | # relative to this directory. They are copied after the builtin static files, 162 | # so a file named "default.css" will overwrite the builtin "default.css". 163 | html_static_path = ['_static'] 164 | 165 | # Custom sidebar templates, must be a dictionary that maps document names 166 | # to template names. 167 | # 168 | # The default sidebars (for documents that don't match any pattern) are 169 | # defined by theme itself. Builtin themes are using these templates by 170 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 171 | # 'searchbox.html']``. 172 | # 173 | html_sidebars = { 174 | '**': ( 175 | 'sidebar/brand.html', 176 | 'project-description.html', 177 | 'sidebar/search.html', 178 | 'sidebar/scroll-start.html', 179 | 'sidebar/navigation.html', 180 | 'github-sponsors.html', 181 | 'sidebar/ethical-ads.html', 182 | 'sidebar/scroll-end.html', 183 | ), 184 | } 185 | 186 | 187 | # -- Options for HTMLHelp output --------------------------------------------- 188 | 189 | # Output file base name for HTML help builder. 190 | htmlhelp_basename = f'{project}doc' 191 | 192 | 193 | # -- Options for LaTeX output ------------------------------------------------ 194 | 195 | latex_elements = { 196 | # The paper size ('letterpaper' or 'a4paper'). 197 | # 198 | # 'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | # 202 | # 'pointsize': '10pt', 203 | 204 | # Additional stuff for the LaTeX preamble. 205 | # 206 | # 'preamble': '', 207 | 208 | # Latex figure (float) alignment 209 | # 210 | # 'figure_align': 'htbp', 211 | } 212 | 213 | # Grouping the document tree into LaTeX files. List of tuples 214 | # (source start file, target name, title, 215 | # author, documentclass [howto, manual, or own class]). 216 | latex_documents = [ 217 | ( 218 | master_doc, f'{project}.tex', f'{project} Documentation', 219 | author, 'manual', 220 | ), 221 | ] 222 | 223 | 224 | # -- Options for manual page output ------------------------------------------ 225 | 226 | # One entry per manual page. List of tuples 227 | # (source start file, name, description, authors, manual section). 228 | man_pages = [ 229 | ( 230 | master_doc, project, f'{project} Documentation', 231 | [author], 1, 232 | ), 233 | ] 234 | 235 | 236 | # -- Options for Texinfo output ---------------------------------------------- 237 | 238 | # Grouping the document tree into Texinfo files. List of tuples 239 | # (source start file, target name, title, author, 240 | # dir menu entry, description, category) 241 | texinfo_documents = [ 242 | ( 243 | master_doc, project, f'{project} Documentation', 244 | author, project, 'One line description of project.', 245 | 'Miscellaneous', 246 | ), 247 | ] 248 | 249 | 250 | # -- Options for Epub output ------------------------------------------------- 251 | 252 | # Bibliographic Dublin Core info. 253 | epub_title = project 254 | epub_author = author 255 | epub_publisher = author 256 | epub_copyright = copyright 257 | 258 | # The unique identifier of the text. This can be a ISBN number 259 | # or the project homepage. 260 | # 261 | # epub_identifier = '' 262 | 263 | # A unique identification for the text. 264 | # 265 | # epub_uid = '' 266 | 267 | # A list of files that should not be packed into the epub file. 268 | epub_exclude_files = ['search.html'] 269 | 270 | 271 | # -- Extension configuration ------------------------------------------------- 272 | 273 | # -- Options for intersphinx extension --------------------------------------- 274 | 275 | # Example configuration for intersphinx: refer to the Python standard library. 276 | intersphinx_mapping = { 277 | 'gidgethub': ('https://gidgethub.readthedocs.io/en/latest/', None), 278 | 'python': ('https://docs.python.org/', None), 279 | 'tutorial': ('https://tutorial.octomachinery.dev/en/latest/', None), 280 | } 281 | 282 | # -- Options for todo extension ---------------------------------------------- 283 | 284 | # If true, `todo` and `todoList` produce output, else they produce nothing. 285 | todo_include_todos = True 286 | 287 | 288 | def set_up_template_context( 289 | app, pagename, templatename, # pylint: disable=unused-argument 290 | context, 291 | doctree, # pylint: disable=unused-argument 292 | ): 293 | """Add a dist summary to Jinja2 context.""" 294 | context['theme_prj_summary'] = PRJ_SUMMARY 295 | 296 | 297 | def setup(app): 298 | """Patch the sphinx theme set up stage.""" 299 | app.connect('html-page-context', set_up_template_context) 300 | 301 | 302 | # Ref: https://github.com/python-attrs/attrs/pull/571/files\ 303 | # #diff-85987f48f1258d9ee486e3191495582dR82 304 | default_role = 'any' 305 | -------------------------------------------------------------------------------- /octomachinery/github/models/events.py: -------------------------------------------------------------------------------- 1 | """Generic GitHub event containers.""" 2 | 3 | from __future__ import annotations 4 | 5 | import json 6 | import pathlib 7 | import uuid 8 | import warnings 9 | from typing import ( 10 | TYPE_CHECKING, Any, Iterable, Mapping, TextIO, Type, Union, cast, 11 | ) 12 | 13 | from gidgethub.sansio import Event as _GidgetHubEvent 14 | 15 | import attr 16 | 17 | # pylint: disable=relative-beyond-top-level 18 | from ...utils.asynctools import aio_gather 19 | # pylint: disable=relative-beyond-top-level 20 | from ..utils.event_utils import ( 21 | augment_http_headers, parse_event_stub_from_fd, validate_http_headers, 22 | ) 23 | 24 | 25 | if TYPE_CHECKING: 26 | # pylint: disable=relative-beyond-top-level 27 | from ...app.routing.abc import OctomachineryRouterBase 28 | 29 | 30 | __all__ = 'GitHubEvent', 'GitHubWebhookEvent' 31 | 32 | 33 | def _to_uuid4(value: Union[str, uuid.UUID]) -> uuid.UUID: 34 | """Return a UUID from the value.""" 35 | if isinstance(value, uuid.UUID): 36 | return value 37 | 38 | return uuid.UUID(value, version=4) 39 | 40 | 41 | def _to_dict(value: Union[Mapping[str, Any], bytes, str]) -> Mapping[str, Any]: 42 | """Return a dict from the value.""" 43 | if isinstance(value, dict): 44 | return value 45 | 46 | if isinstance(value, bytes): 47 | value = value.decode() 48 | 49 | return json.loads(cast(str, value)) 50 | 51 | 52 | @attr.dataclass(frozen=True) 53 | class GitHubEvent: 54 | """Representation of a generic source-agnostic GitHub event.""" 55 | 56 | name: str 57 | """Event name.""" 58 | payload: Mapping[str, Any] = attr.ib(converter=_to_dict) 59 | """Event payload object.""" 60 | 61 | @payload.validator 62 | def _is_payload_dict( 63 | self, attribute: str, value: Mapping[str, Any], 64 | ) -> None: 65 | """Verify that the attribute value is a dict. 66 | 67 | :raises ValueError: if it's not 68 | """ 69 | if isinstance(value, dict): 70 | return 71 | 72 | raise ValueError( 73 | f'{value!r} is passed as {attribute!s} but it must ' 74 | 'be an instance of dict', 75 | ) 76 | 77 | @classmethod 78 | def from_file( 79 | cls: Type[GitHubEvent], 80 | event_name: str, 81 | event_path: Union[pathlib.Path, str], 82 | ) -> GitHubEvent: 83 | """Construct a GitHubEvent instance from event name and file.""" 84 | # NOTE: This could be async but it probably doesn't matter 85 | # NOTE: since it's called just once during init and GitHub 86 | # NOTE: Action runtime only has one event to process 87 | # NOTE: OTOH it may slow-down tests parallelism 88 | # NOTE: so may deserve to be fixed 89 | with pathlib.Path(event_path).open(encoding='utf-8') as event_source: 90 | return cls(event_name, json.load(event_source)) 91 | 92 | @classmethod 93 | def from_fixture_fd( 94 | cls: Type[GitHubEvent], 95 | event_fixture_fd: TextIO, 96 | *, 97 | event: Union[str, None] = None, 98 | ) -> GitHubEvent: 99 | """Make a GitHubEvent from a fixture fd and an optional name.""" 100 | headers, payload = parse_event_stub_from_fd(event_fixture_fd) 101 | if event and 'x-github-event' in headers: 102 | raise ValueError( 103 | 'Supply only one of an event name ' 104 | 'or an event header in the fixture file', 105 | ) 106 | event_name = event or headers['x-github-event'] 107 | return cls(event_name, payload) 108 | 109 | @classmethod 110 | def from_fixture( 111 | cls: Type[GitHubEvent], 112 | event_fixture_path: Union[pathlib.Path, str], 113 | *, 114 | event: Union[str, None] = None, 115 | ) -> GitHubEvent: 116 | """Make a GitHubEvent from a fixture and an optional name.""" 117 | with pathlib.Path( 118 | event_fixture_path, 119 | ).open(encoding='utf-8') as event_source: 120 | return cls.from_fixture_fd(event_source, event=event) 121 | 122 | @classmethod 123 | def from_gidgethub(cls, event: _GidgetHubEvent) -> GitHubEvent: 124 | """Construct GitHubEvent from from GidgetHub Event.""" 125 | return cls( 126 | name=event.event, 127 | payload=event.data, 128 | ) 129 | 130 | def to_gidgethub(self) -> _GidgetHubEvent: 131 | """Produce GidgetHub Event from self.""" 132 | return _GidgetHubEvent( 133 | data=self.payload, 134 | event=self.name, 135 | delivery_id=str(uuid.uuid4()), 136 | ) 137 | 138 | async def dispatch_via( 139 | self, 140 | *routers: OctomachineryRouterBase, 141 | ctx: Union[Mapping[str, Any], None] = None, 142 | ) -> Iterable[Any]: 143 | """Invoke this event handlers from different routers.""" 144 | if not routers: 145 | raise ValueError('At least one router must be supplied') 146 | 147 | if ctx is None: 148 | ctx = {} 149 | 150 | return await aio_gather( 151 | *( 152 | r.dispatch(self, **ctx) 153 | for r in routers 154 | ), 155 | ) 156 | 157 | 158 | @attr.dataclass(frozen=True) 159 | class GitHubWebhookEvent(GitHubEvent): 160 | """Representation of a GitHub event arriving by HTTP.""" 161 | 162 | delivery_id: uuid.UUID = attr.ib(converter=_to_uuid4) 163 | """A unique UUID4 identifier of the event delivery on GH side.""" 164 | 165 | @delivery_id.validator 166 | def _is_delivery_id(self, attribute: str, value: uuid.UUID) -> None: 167 | """Verify that the attribute value is UUID v4. 168 | 169 | :raises ValueError: if it's not 170 | """ 171 | if isinstance(value, uuid.UUID) and value.version == 4: 172 | return 173 | 174 | raise ValueError( 175 | f'{value!r} is passed as {attribute!s} but it must ' 176 | 'be an instance of UUID v4', 177 | ) 178 | 179 | @classmethod 180 | def from_file( 181 | cls: Type[GitHubWebhookEvent], 182 | event_name: str, 183 | event_path: Union[pathlib.Path, str], 184 | ) -> GitHubWebhookEvent: 185 | """Explode when constructing from file.""" 186 | raise RuntimeError( 187 | 'Webhook event is not supposed to be constructed from a file', 188 | ) 189 | 190 | @classmethod 191 | def from_fixture_fd( 192 | cls: Type[GitHubWebhookEvent], 193 | event_fixture_fd: TextIO, 194 | *, 195 | event: Union[str, None] = None, 196 | ) -> GitHubWebhookEvent: 197 | """Make GitHubWebhookEvent from fixture fd and optional name.""" 198 | headers, payload = parse_event_stub_from_fd(event_fixture_fd) 199 | if event and 'x-github-event' in headers: 200 | raise ValueError( 201 | 'Supply only one of an event name ' 202 | 'or an event header in the fixture file', 203 | ) 204 | headers['x-github-event'] = event or headers['x-github-event'] 205 | headers = augment_http_headers(headers) 206 | validate_http_headers(headers) 207 | 208 | return cls( 209 | name=headers['x-github-event'], 210 | payload=payload, 211 | delivery_id=headers['x-github-delivery'], 212 | ) 213 | 214 | @classmethod 215 | def from_fixture( 216 | cls: Type[GitHubWebhookEvent], 217 | event_fixture_path: Union[pathlib.Path, str], 218 | *, 219 | event: Union[str, None] = None, 220 | ) -> GitHubWebhookEvent: 221 | """Make a GitHubWebhookEvent from fixture and optional name.""" 222 | with pathlib.Path( 223 | event_fixture_path, 224 | ).open(encoding='utf-8') as event_source: 225 | return cls.from_fixture_fd(event_source, event=event) 226 | 227 | @classmethod 228 | def from_http_request( 229 | cls: Type[GitHubWebhookEvent], 230 | http_req_headers: Mapping[str, str], 231 | http_req_body: bytes, 232 | ): 233 | """Make a GitHubWebhookEvent from HTTP req headers and body.""" 234 | return cls( 235 | name=http_req_headers['x-github-event'], 236 | payload=json.loads(http_req_body.decode()), 237 | delivery_id=http_req_headers['x-github-delivery'], 238 | ) 239 | 240 | @classmethod 241 | def from_gidgethub(cls, event: _GidgetHubEvent) -> GitHubWebhookEvent: 242 | """Construct GitHubWebhookEvent from from GidgetHub Event.""" 243 | return cls( 244 | name=event.event, 245 | payload=event.data, 246 | delivery_id=event.delivery_id, 247 | ) 248 | 249 | def to_gidgethub(self) -> _GidgetHubEvent: 250 | """Produce GidgetHub Event from self.""" 251 | return _GidgetHubEvent( 252 | data=self.payload, 253 | event=self.name, # pylint: disable=no-member 254 | delivery_id=self.delivery_id, 255 | ) 256 | 257 | 258 | class GidgetHubEventMixin: 259 | """A temporary shim for GidgetHub event interfaces. 260 | 261 | It's designed to be used during the refactoring period when interfacing 262 | with the new event representation layer :py:class:`~GitHubEvent`. 263 | """ 264 | 265 | @property 266 | def data(self): 267 | """Event payload dict alias.""" 268 | warnings.warn( 269 | "Relying on GidgetHub's event class interfaces will be deprecated " 270 | "in the future releases. Please use 'payload' attribute to access " 271 | 'the event name instead.', 272 | category=PendingDeprecationWarning, 273 | stacklevel=2, 274 | ) 275 | # pylint: disable=fixme 276 | return self.payload # type: ignore[attr-defined] # FIXME 277 | 278 | @property 279 | def event(self): 280 | """Event name alias.""" 281 | warnings.warn( 282 | "Relying on GidgetHub's event class interfaces will be deprecated " 283 | "in the future releases. Please use 'name' attribute to access " 284 | 'the event name instead.', 285 | category=PendingDeprecationWarning, 286 | stacklevel=2, 287 | ) 288 | # pylint: disable=fixme 289 | return self.name # type: ignore[attr-defined] # FIXME 290 | 291 | 292 | class GidgetHubActionEvent(GidgetHubEventMixin, GitHubEvent): 293 | """GitHub Action event wrapper exposing GidgetHub attrs.""" 294 | 295 | 296 | class GidgetHubWebhookEvent(GidgetHubEventMixin, GitHubWebhookEvent): 297 | """GitHub HTTP event wrapper exposing GidgetHub attrs.""" 298 | -------------------------------------------------------------------------------- /tests/github/utils/event_utils_test.py: -------------------------------------------------------------------------------- 1 | """Tests for CLI utility functions.""" 2 | 3 | from io import StringIO 4 | from textwrap import dedent 5 | from uuid import uuid1, uuid4 6 | 7 | import multidict 8 | 9 | import pytest 10 | 11 | from octomachinery.github.utils.event_utils import ( 12 | _probe_json, _transform_http_headers_list_to_multidict, 13 | augment_http_headers, make_http_headers_from_event, 14 | parse_event_stub_from_fd, validate_http_headers, 15 | ) 16 | 17 | 18 | UNCHANGED_UUID4_STR = str(uuid4()) 19 | 20 | 21 | @pytest.mark.parametrize( 22 | 'vcr_contents, vcr_headers', 23 | ( 24 | pytest.param( 25 | dedent( 26 | """ 27 | --- 28 | # This needs to be a sequence of mappings; 29 | # it cannot be just a mapping because headers 30 | # may occur multiple times 31 | - Content-Type: application/json 32 | - X-GitHub-Delivery: 2791443c-641a-40fa-836d-031a26f0d45f 33 | - X-GitHub-Event: ping 34 | --- 35 | { 36 | "hook": {"app_id": 0}, 37 | "hook_id": 0, 38 | "zen": "Hey zen!" 39 | } 40 | """, 41 | ), 42 | multidict.CIMultiDict({ 43 | 'Content-Type': 'application/json', 44 | 'X-GitHub-Delivery': '2791443c-641a-40fa-836d-031a26f0d45f', 45 | 'X-GitHub-Event': 'ping', 46 | }), 47 | id='YAML', 48 | ), 49 | pytest.param( 50 | # pylint: disable=line-too-long 51 | """ 52 | [{"Content-Type": "application/json"}, {"X-GitHub-Delivery": "2791443c-641a-40fa-836d-031a26f0d45f"}, {"X-GitHub-Event": "ping"}] 53 | {"hook": {"app_id": 0}, "hook_id": 0, "zen": "Hey zen!"} 54 | """.strip(), # noqa: E501 55 | multidict.CIMultiDict({ 56 | 'Content-Type': 'application/json', 57 | 'X-GitHub-Delivery': '2791443c-641a-40fa-836d-031a26f0d45f', 58 | 'X-GitHub-Event': 'ping', 59 | }), 60 | id='JSONL', 61 | ), 62 | pytest.param( 63 | '{\n"hook": {\n"app_id": 0}, "hook_id": 0, "zen": "Hey zen!"}', 64 | multidict.CIMultiDict(), 65 | id='JSON', 66 | ), 67 | pytest.param( 68 | """ 69 | {"hook": {"app_id": 0}, "hook_id": 0, "zen": "Hey zen!"} 70 | { 71 | """.strip(), 72 | multidict.CIMultiDict(), 73 | id='JSONL with the broken 2nd line', 74 | ), 75 | ), 76 | ) 77 | def test_parse_event_stub_from_fd(vcr_contents, vcr_headers): 78 | """Check that all of YAML, JSONL and JSON VCR modes are loadable.""" 79 | vcr_event = { 80 | 'hook': { 81 | 'app_id': 0, 82 | }, 83 | 'hook_id': 0, 84 | 'zen': 'Hey zen!', 85 | } 86 | with StringIO(vcr_contents) as event_file_fd: 87 | actual_parsed_vcr = parse_event_stub_from_fd(event_file_fd) 88 | 89 | expected_parsed_vcr = vcr_headers, vcr_event 90 | assert actual_parsed_vcr == expected_parsed_vcr 91 | 92 | 93 | @pytest.mark.parametrize( 94 | 'vcr_contents', 95 | ( 96 | pytest.param( 97 | dedent( 98 | """ 99 | --- 100 | - X-GitHub-Event: ping 101 | --- 102 | {} 103 | --- 104 | - extra document 105 | """, 106 | ), 107 | id='broken 3-document YAML (must be 2 documents)', 108 | ), 109 | pytest.param( 110 | dedent( 111 | """ 112 | --- 113 | """, 114 | ), 115 | id='broken empty-document YAML', 116 | ), 117 | pytest.param( 118 | """ 119 | [{"X-GitHub-Event": "ping"}] 120 | {"hook": {"app_id": 0}, "hook_id": 0, "zen": "Hey zen!"} 121 | {} 122 | """, 123 | id='broken 3-line JSONL (must be 2 lines)', 124 | ), 125 | pytest.param( 126 | '1{"hook": {"app_id": 0}, "hook_id": 0, "zen": "Hey zen!"}', 127 | id='broken JSON syntax (leading garbage)', 128 | ), 129 | ), 130 | ) 131 | def test_parse_event_stub_from_fd__invalid(vcr_contents): 132 | """Verify that feeding unconventional VCRs raises ValueError.""" 133 | expected_error_message = ( 134 | r'^The input event VCR file has invalid structure\. ' 135 | r'It must be either of YAML, JSONL or JSON\.$' 136 | ) 137 | with StringIO(vcr_contents) as file_descr, pytest.raises( 138 | ValueError, 139 | match=expected_error_message, 140 | ): 141 | parse_event_stub_from_fd(file_descr) 142 | 143 | 144 | @pytest.mark.parametrize( 145 | 'http_headers', 146 | ( 147 | pytest.param( 148 | multidict.CIMultiDict({ 149 | 'Content-Type': 'application/json', 150 | 'User-Agent': 'GitHub-Hookshot/dict-test', 151 | 'X-GitHub-Delivery': str(uuid4()), 152 | 'X-GitHub-Event': 'issue', 153 | }), 154 | id='multidict', 155 | ), 156 | pytest.param( 157 | { 158 | 'content-type': 'application/json', 159 | 'user-agent': 'GitHub-Hookshot/dict-test', 160 | 'x-github-delivery': str(uuid4()), 161 | 'x-github-event': 'pull_request', 162 | }, 163 | id='dict', 164 | ), 165 | pytest.param( 166 | make_http_headers_from_event('ping'), 167 | id='make_http_headers_from_event constructor', 168 | ), 169 | ), 170 | ) 171 | def test_validate_http_headers(http_headers): 172 | """Verify that valid headers collections don't raise exceptions.""" 173 | assert validate_http_headers(http_headers) is None # no exceptions raised 174 | 175 | 176 | @pytest.mark.parametrize( 177 | 'http_headers, error_message', 178 | ( 179 | pytest.param( 180 | { 181 | 'content-type': 'multipart/form-data', 182 | }, 183 | r"^Content\-Type must be 'application\/json'$", 184 | id='Content-Type', 185 | ), 186 | pytest.param( 187 | { 188 | 'content-type': 'application/json', 189 | 'user-agent': 'Fake-GitHub-Hookshot/dict-test', 190 | }, 191 | r"^User\-Agent must start with 'GitHub-Hookshot\/'$", 192 | id='User-Agent', 193 | ), 194 | pytest.param( 195 | { 196 | 'content-type': 'application/json', 197 | 'user-agent': 'GitHub-Hookshot/dict-test', 198 | 'x-github-delivery': 'garbage', 199 | }, 200 | r'^X\-GitHub\-Delivery must be of type UUID4$', 201 | id='garbage X-GitHub-Delivery', 202 | ), 203 | pytest.param( 204 | { 205 | 'content-type': 'application/json', 206 | 'user-agent': 'GitHub-Hookshot/dict-test', 207 | 'x-github-delivery': str(uuid1()), 208 | }, 209 | r'^X\-GitHub\-Delivery must be of type UUID4$', 210 | id='UUID1 X-GitHub-Delivery', 211 | ), 212 | pytest.param( 213 | { 214 | 'content-type': 'application/json', 215 | 'user-agent': 'GitHub-Hookshot/dict-test', 216 | 'x-github-delivery': str(uuid4()), 217 | 'x-github-event': None, 218 | }, 219 | r'^X\-GitHub\-Event must be a string$', 220 | id='X-GitHub-Event', 221 | ), 222 | ), 223 | ) 224 | def test_validate_http_headers__invalid(http_headers, error_message): 225 | """Check that invalid headers cause ValueError.""" 226 | with pytest.raises(ValueError, match=error_message): 227 | validate_http_headers(http_headers) 228 | 229 | 230 | @pytest.mark.parametrize( 231 | 'incomplete_http_headers, expected_headers', 232 | ( 233 | pytest.param( 234 | { 235 | 'content-type': 'application/json', 236 | 'user-agent': 'GitHub-Hookshot/dict-test', 237 | 'x-github-delivery': UNCHANGED_UUID4_STR, 238 | 'x-github-event': 'pull_request', 239 | 'x-header': 'x_value', 240 | }, 241 | { 242 | 'content-type': 'application/json', 243 | 'user-agent': 'GitHub-Hookshot/dict-test', 244 | 'x-github-delivery': UNCHANGED_UUID4_STR, 245 | 'x-github-event': 'pull_request', 246 | 'x-header': 'x_value', 247 | }, 248 | id='unchanged', 249 | ), 250 | pytest.param( 251 | { 252 | 'user-agent': 'GitHub-Hookshot/dict-test', 253 | 'x-github-delivery': str(uuid4()), 254 | 'x-github-event': 'pull_request', 255 | 'x-header': 'x_value', 256 | }, 257 | { 258 | 'content-type': 'application/json', 259 | 'user-agent': 'GitHub-Hookshot/dict-test', 260 | 'x-github-event': 'pull_request', 261 | 'x-header': 'x_value', 262 | }, 263 | id='Content-Type', 264 | ), 265 | pytest.param( 266 | { 267 | 'content-type': 'application/json', 268 | 'x-github-delivery': str(uuid4()), 269 | 'x-github-event': 'pull_request', 270 | 'x-header': 'x_value', 271 | }, 272 | { 273 | 'content-type': 'application/json', 274 | 'user-agent': 'GitHub-Hookshot/fallback-value', 275 | 'x-github-event': 'pull_request', 276 | 'x-header': 'x_value', 277 | }, 278 | id='User-Agent', 279 | ), 280 | pytest.param( 281 | { 282 | 'content-type': 'application/json', 283 | 'user-agent': 'GitHub-Hookshot/dict-test', 284 | 'x-github-event': 'pull_request', 285 | 'x-header': 'x_value', 286 | }, 287 | { 288 | 'content-type': 'application/json', 289 | 'user-agent': 'GitHub-Hookshot/dict-test', 290 | 'x-github-event': 'pull_request', 291 | 'x-header': 'x_value', 292 | }, 293 | id='X-GitHub-Delivery', 294 | ), 295 | pytest.param( 296 | { 297 | 'x-github-event': 'ping', 298 | }, 299 | { 300 | 'content-type': 'application/json', 301 | 'user-agent': 'GitHub-Hookshot/fallback-value', 302 | 'x-github-event': 'ping', 303 | }, 304 | id='X-GitHub-Event', 305 | ), 306 | ), 307 | ) 308 | def test_augment_http_headers(incomplete_http_headers, expected_headers): 309 | """Check that mandatory headers are present after augmentation.""" 310 | augmented_headers = augment_http_headers(incomplete_http_headers) 311 | 312 | assert validate_http_headers(augmented_headers) is None 313 | 314 | original_event = incomplete_http_headers['x-github-event'] 315 | assert augmented_headers['x-github-event'] == original_event 316 | 317 | for header_name, header_value in expected_headers.items(): 318 | assert augmented_headers[header_name] == header_value 319 | 320 | 321 | def test_make_http_headers_from_event(): 322 | """Smoke-test fake HTTP headers constructor.""" 323 | event_name = 'issue_comment' 324 | http_headers = make_http_headers_from_event(event_name) 325 | 326 | assert http_headers['X-GitHub-Event'] == event_name 327 | assert http_headers['User-Agent'].endswith('/fallback-value') 328 | assert validate_http_headers(http_headers) is None 329 | 330 | 331 | def test__transform_http_headers_list_to_multidict__invalid(): 332 | """Check the headers format validation.""" 333 | error_message = ( 334 | '^Headers must be a sequence of mappings ' 335 | 'because keys can repeat$' 336 | ) 337 | with pytest.raises(ValueError, match=error_message): 338 | _transform_http_headers_list_to_multidict({}) 339 | 340 | 341 | def test__probe_json(): 342 | """Test that JSON probe loads mappings.""" 343 | vcr_contents = ( 344 | '{\n"hook": {\n"app_id": 0},' 345 | ' "hook_id": 0, ' 346 | '"zen": "Hey zen!"}' 347 | ) 348 | vcr_headers = () 349 | vcr_event = { 350 | 'hook': { 351 | 'app_id': 0, 352 | }, 353 | 'hook_id': 0, 354 | 'zen': 'Hey zen!', 355 | } 356 | with StringIO(vcr_contents) as event_file_fd: 357 | actual_parsed_vcr = _probe_json(event_file_fd) 358 | 359 | expected_parsed_vcr = vcr_headers, vcr_event 360 | assert actual_parsed_vcr == expected_parsed_vcr 361 | 362 | 363 | def test__probe_json__invalid(): 364 | """Verify that non-mapping objects crash pure JSON probe.""" 365 | expected_error_message = '^JSON file must only contain an object mapping$' 366 | with StringIO('[]') as file_descr, pytest.raises( 367 | ValueError, 368 | match=expected_error_message, 369 | ): 370 | _probe_json(file_descr) 371 | --------------------------------------------------------------------------------