├── __init__.py ├── bin ├── __init__.py ├── check_mta_sts.py └── add_users_all_org_members_github_team.py ├── test ├── __init__.py ├── test_bin │ ├── __init__.py │ ├── test_check_mta_sts.py │ └── test_add_users_all_org_members_github_team.py ├── test_clients │ └── __init__.py ├── test_services │ ├── __init__.py │ ├── test_s3_service.py │ └── test_slack_service.py ├── fixtures │ └── test_data.csv ├── files │ └── test_config.py └── test_utils │ └── test_environment.py ├── utils ├── __init__.py └── environment.py ├── clients └── __init__.py ├── services ├── __init__.py ├── secret_manager_service.py ├── s3_service.py ├── route53_service.py ├── slack_service.py └── github_service.py ├── .coveragerc ├── .shellcheckrc ├── .gitleaksignore ├── .github ├── CODEOWNERS ├── workflows │ ├── job-alarm-for-new-secret-scanning-alert.yml │ ├── experimental-check-version-pinning.yml │ ├── cicd-dependency-review.yml │ ├── cicd-tests.yml │ ├── job-add-github-members-to-root-team-moj.yml │ ├── job-add-github-members-to-root-team-mojas.yml │ ├── job-identify-dormant-github-users-v2.yml │ └── cicd-mega-linter.yml ├── ISSUE_TEMPLATE │ ├── firebreak-template.md │ ├── trivy-vulnerability-template.md │ └── operations-engineering-story.md └── pull_request_template.md ├── terrascan-config.toml ├── .yamllint ├── .pre-commit-config.yaml ├── config ├── logging_config.py └── constants.py ├── .gitignore ├── Makefile ├── .flake8 ├── terraform └── dsd │ └── iam │ └── deprecated │ ├── dsd_route53_read_role.tf │ ├── github_oidc.tf │ └── octodns_dns_write.tf ├── Pipfile ├── LICENSE ├── .mega-linter.yml ├── README.md └── .pylintrc /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /clients/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_clients/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_also = 3 | main() 4 | -------------------------------------------------------------------------------- /.shellcheckrc: -------------------------------------------------------------------------------- 1 | disable=SC2086 # Disables SC2086: Double quote to prevent globbing and word splitting. 2 | -------------------------------------------------------------------------------- /.gitleaksignore: -------------------------------------------------------------------------------- 1 | 98149582bcbb15fcf92c77b7f7d4b741750fdcce:.github/workflows/cicd-terraform-github-repos.yml:generic-api-key:57 -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @ministryofjustice/operations-engineering 2 | 3 | /data/* @ministryofjustice/operations-engineering-support 4 | -------------------------------------------------------------------------------- /terrascan-config.toml: -------------------------------------------------------------------------------- 1 | # Scan and skip rules configuration 2 | [rules] 3 | skip-rules = [ 4 | "AC_GITHUB_0002" 5 | ] 6 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | yaml-files: 4 | - '*.yaml' 5 | - '*.yml' 6 | - '.yamllint' 7 | 8 | rules: 9 | document-end: disable 10 | document-start: disable 11 | truthy: disable -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | exclude: ^tests/fixtures/ 9 | -------------------------------------------------------------------------------- /test/fixtures/test_data.csv: -------------------------------------------------------------------------------- 1 | Type,Action,Date 2 | GitHub,GitHub – add user to org,2024-07-22 3 | GitHub,GitHub – add user to org,2024-07-22 4 | GitHub,GitHub – remove user from org,2024-07-22 5 | 1Password,1Password - information/help,2024-07-22 6 | API,API Key,2024-07-22 7 | DNS,DNS/Domain,2024-07-22 8 | Other,Tools Information/help,2024-07-22 9 | Other,Refer to another team,2024-07-22 -------------------------------------------------------------------------------- /config/logging_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | LOGGING_LEVEL = "" 5 | 6 | if (LOGGING_LEVEL := os.getenv('LOGGING_LEVEL')) not in ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET']: 7 | LOGGING_LEVEL = "INFO" 8 | 9 | logging.basicConfig( 10 | format="%(asctime)s %(levelname)-8s %(message)s", 11 | level=LOGGING_LEVEL, 12 | datefmt="%Y-%m-%d %H:%M:%S", 13 | ) 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .venv 3 | venv 4 | env 5 | *.pkg 6 | .vscode 7 | .idea/ 8 | *.pyc 9 | coverage.xml 10 | .coverage 11 | .tox* 12 | htmlcov 13 | *__pycache__* 14 | .DS_Store 15 | .terraform* 16 | codeql* 17 | codeql-custom-queries-python/ 18 | 19 | ministryofjustice_first_email_list.json 20 | moj-analytical-services_first_email_list.json 21 | export-* 22 | export-moj-analytical-services.json 23 | dormant.csv 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | local-setup: 2 | brew install pre-commit && pre-commit install 3 | 4 | redact-terraform-output: 5 | sed -e 's/AWS_SECRET_ACCESS_KEY".*//g' \ 6 | -e 's/AWS_ACCESS_KEY_ID".*//g' \ 7 | -e 's/$AWS_SECRET_ACCESS_KEY".*//g' \ 8 | -e 's/$AWS_ACCESS_KEY_ID".*//g' \ 9 | -e 's/\[id=.*\]/\[id=\]/g' \ 10 | -e 's/::[0-9]\{12\}:/::REDACTED:/g' \ 11 | -e 's/:[0-9]\{12\}:/:REDACTED:/g' 12 | -------------------------------------------------------------------------------- /test/files/test_config.py: -------------------------------------------------------------------------------- 1 | test_config = { 2 | "keys": { 3 | "gandi_api_key": "test_key", 4 | "notify_api_key": "test_key", 5 | }, 6 | "template_ids": { 7 | "cert_expiry": "test_template_id" 8 | }, 9 | "urls": { 10 | "gandi_base_url": "test_base_url", 11 | "gandi_cert_url_extension": "test_cert_url_extension", 12 | }, 13 | "cert_expiry_thresholds": [30, 15, 1], 14 | "reply_email": "test_reply_email", 15 | "gandi": { 16 | "topup_amount": "test_amount" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /.github/workflows/job-alarm-for-new-secret-scanning-alert.yml: -------------------------------------------------------------------------------- 1 | name: Alarm for new secret scanning alerts 2 | 3 | on: 4 | schedule: 5 | - cron: "0 4 * * *" 6 | workflow_dispatch: 7 | 8 | jobs: 9 | detect-secret-scanning-alerts: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 13 | 14 | - uses: ministryofjustice/github-actions/slack-github-secret-scanning-integration@721b0f273fc8356611cb18b3dfc02074d59217da # v18.2.4 15 | with: 16 | github-token: ${{ secrets.OPS_ENG_GENERAL_ADMIN_BOT_PAT }} 17 | slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} 18 | -------------------------------------------------------------------------------- /config/constants.py: -------------------------------------------------------------------------------- 1 | # class Constants: 2 | GITHUB_LICENSE_THRESHOLD = 50 3 | MINISTRY_OF_JUSTICE = "ministryofjustice" 4 | MOJ_ANALYTICAL_SERVICES = "moj-analytical-services" 5 | MINISTRY_OF_JUSTICE_TEST = "ministryofjustice-test" 6 | MISSING_EMAIL_ADDRESS = "-" 7 | NO_ACTIVITY = "no activity" 8 | ENTERPRISE = "ministry-of-justice-uk" 9 | SLACK_CHANNEL = "operations-engineering-alerts" 10 | SR_SLACK_CHANNEL = "operations-engineering-team" 11 | OPERATIONS_ENGINEERING_GITHUB_USERNAMES = [ 12 | "SeanPrivett", 13 | "andyrogers1973", 14 | "tamsinforbes", 15 | "AntonyBishop", 16 | "connormaglynn", 17 | "levgorbunov1", 18 | "moj-operations-engineering-bot", 19 | "jnioche-jd", 20 | ] 21 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # H101: Use TODO(NAME) 3 | # H202: assertRaises Exception too broad 4 | # H233: Python 3.x incompatible use of print operator 5 | # H301: one import per line 6 | # H306: imports not in alphabetical order (time, os) 7 | # H401: docstring should not start with a space 8 | # H403: multi line docstrings should end on a new line 9 | # H404: multi line docstring should start without a leading new line 10 | # H405: multi line docstring summary not separated with an empty line 11 | # H501: Do not use self.__dict__ for string formatting 12 | # E501: line too long 13 | extend-ignore = H101,H202,H233,H301,H306,H401,H403,H404,H405,H501,E501 14 | exclude = 15 | # No need to traverse our git directory 16 | .git, 17 | max-complexity = 10 18 | -------------------------------------------------------------------------------- /.github/workflows/experimental-check-version-pinning.yml: -------------------------------------------------------------------------------- 1 | name: 🧪 Check Version Pinning 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | jobs: 11 | check-version-pinning: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 16 | with: 17 | fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis 18 | 19 | - name: Check for unpinned Actions 20 | uses: ministryofjustice/github-actions/check-version-pinning@ccf9e3a4a828df1ec741f6c8e6ed9d0acaef3490 21 | with: 22 | workflow_directory: ".github/workflows" 23 | scan_mode: "full" 24 | -------------------------------------------------------------------------------- /.github/workflows/cicd-dependency-review.yml: -------------------------------------------------------------------------------- 1 | # Need a GitHub Advanced Security license to run this action on private repos. 2 | 3 | name: ♻️ Dependency Review 4 | on: 5 | pull_request: 6 | types: [opened, edited, reopened, synchronize] 7 | paths-ignore: 8 | - "source/**" 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | dependency-review: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout Repository 18 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 19 | - name: Dependency Review 20 | uses: actions/dependency-review-action@c74b580d73376b7750d3d2a50bfb8adc2c937507 # v3.1.5 21 | with: 22 | # Possible values: critical, high, moderate, low 23 | fail-on-severity: critical 24 | -------------------------------------------------------------------------------- /services/secret_manager_service.py: -------------------------------------------------------------------------------- 1 | import json 2 | import botocore.session 3 | from botocore.exceptions import ClientError 4 | from aws_secretsmanager_caching import SecretCache, SecretCacheConfig 5 | 6 | 7 | class SecretsManagerService: 8 | def __init__(self) -> None: 9 | client = botocore.session.get_session().create_client('secretsmanager') 10 | cache_config = SecretCacheConfig() 11 | self.cache = SecretCache(config=cache_config, client=client) 12 | 13 | def get_secret_set(self, secret_set_name: str): 14 | try: 15 | secret_set = json.loads(self.cache.get_secret_string(secret_set_name)) 16 | return secret_set 17 | except ClientError as error: 18 | print(f"Error retrieving secret set: {error}") 19 | return None 20 | -------------------------------------------------------------------------------- /terraform/dsd/iam/deprecated/dsd_route53_read_role.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "dsd_route53_read_role" { 2 | name = "dsd_route53_read_role" 3 | assume_role_policy = data.aws_iam_policy_document.github_actions_assume_role_policy_document.json 4 | } 5 | 6 | data "aws_iam_policy" "dsd_route53_read_role_policy" { 7 | arn = "arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess" 8 | } 9 | 10 | resource "aws_iam_role_policy_attachment" "dsd_route53_read_role_policy_attatchment" { 11 | role = aws_iam_role.dsd_route53_read_role.name 12 | policy_arn = data.aws_iam_policy.dsd_route53_read_role_policy.arn 13 | } 14 | 15 | resource "github_actions_secret" "dsd_route53_read_role_arn" { 16 | repository = "operations-engineering" 17 | secret_name = "DSD_ROUTE53_READ_ROLE_ARN" 18 | plaintext_value = aws_iam_role.dsd_route53_read_role.arn 19 | } 20 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [scripts] 7 | list_outdated = "pip list --outdated" 8 | tests = "coverage run -m unittest" 9 | tests_report = "coverage report --omit=./test/** --sort=cover --show-missing --skip-empty" 10 | 11 | [packages] 12 | aiohttp = "==3.9.4" 13 | aws_secretsmanager_caching = "==1.1.3" 14 | boto3 = "==1.34.84" 15 | cryptography = "==42.0.5" 16 | gql = "==3.5.0" 17 | notifications-python-client = "==9.0.0" 18 | pandas = "==2.2.2" 19 | pyaml-env = "==1.2.1" 20 | pygithub = "==2.3.0" 21 | pyjwt = "==2.8.0" 22 | python-dateutil = "==2.8.2" 23 | requests = "==2.31.0" 24 | slack-sdk = "==3.27.1" 25 | toml = "==0.10.2" 26 | pyyaml = "==6.0.2" 27 | 28 | [dev-packages] 29 | coverage = "==7.4.4" 30 | freezegun = "==1.4.0" 31 | moto = "==5.0.5" 32 | pytest = "==8.1.1" 33 | requests-mock = "==1.12.1" 34 | 35 | [requires] 36 | python_version = "3.11" 37 | -------------------------------------------------------------------------------- /terraform/dsd/iam/deprecated/github_oidc.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | oidc_provider = "token.actions.githubusercontent.com" 3 | } 4 | 5 | data "aws_iam_openid_connect_provider" "github" { 6 | url = "https://${local.oidc_provider}" 7 | } 8 | 9 | data "aws_iam_policy_document" "github_actions_assume_role_policy_document" { 10 | version = "2012-10-17" 11 | 12 | statement { 13 | effect = "Allow" 14 | actions = ["sts:AssumeRoleWithWebIdentity"] 15 | 16 | principals { 17 | type = "Federated" 18 | identifiers = [data.aws_iam_openid_connect_provider.github.arn] 19 | } 20 | condition { 21 | test = "StringLike" 22 | variable = "${local.oidc_provider}:sub" 23 | values = [ 24 | "repo:ministryofjustice/operations-engineering:*", 25 | "repo:ministryofjustice/dns:*" 26 | ] 27 | } 28 | 29 | condition { 30 | test = "StringEquals" 31 | variable = "${local.oidc_provider}:aud" 32 | values = ["sts.amazonaws.com"] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/firebreak-template.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Firebreak Story Template 3 | about: Create new Firebreak story 4 | title: 'FIREBREAK:' 5 | labels: Firebreak 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Background 11 | 12 | 13 | 14 | ## Questions / Assumptions 15 | 16 | 17 | 18 | ## What hypothesis do we want to test?/What do we want to learn? 19 | 20 | 21 | 22 | ## Definition of done 23 | 24 | 25 | 26 | - [ ] Firebreak finding documented appropriately 27 | - [ ] Demo completed 28 | - [ ] Decision made on whether to progress Firebreak work 29 | - [ ] Does next steps require User Research? 30 | - [ ] Firebreak next step Issues created 31 | - [ ] New Issues referenced in this story before closure 32 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## :eyes: Purpose 4 | 5 | • 6 | 7 | 8 | ## :recycle: What's Changed 9 | 10 | • 11 | 12 | 13 | ## :memo: Notes 14 | 15 | • 16 | 17 | --- 18 | 19 | ### :white_check_mark: Things to Check (Optional) 20 | 21 | - [ ] I have run all unit tests, and they pass. 22 | - [ ] I have ensured my code follows the project's coding standards. 23 | - [ ] I have checked that all new dependencies are up to date and necessary. 24 | -------------------------------------------------------------------------------- /utils/environment.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from dataclasses import dataclass, field 4 | from typing import Dict, List, Optional 5 | 6 | logging.basicConfig(level=logging.INFO) 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | @dataclass 11 | class EnvironmentVariables: 12 | required_vars: List[str] 13 | env_vars: Dict[str, Optional[str]] = field(init=False) 14 | 15 | def __post_init__(self): 16 | self.env_vars = {} 17 | missing_vars = [] 18 | 19 | for var in self.required_vars: 20 | value = os.environ.get(var) 21 | if value is None: 22 | missing_vars.append(var) 23 | logger.error("%s is not set or empty", var) 24 | self.env_vars[var] = value 25 | 26 | if missing_vars: 27 | raise EnvironmentError( 28 | f"Missing required environment variables: {', '.join(missing_vars)}") 29 | 30 | def get(self, var_name: str) -> Optional[str]: 31 | """Retrieve the value of a specific environment variable.""" 32 | return self.env_vars.get(var_name) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License(MIT) 2 | 3 | Copyright(C) 2022-2023 Crown copyright(Ministry of Justice) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files(the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and / or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/cicd-tests.yml: -------------------------------------------------------------------------------- 1 | name: ♻️ Tests 2 | on: 3 | push: 4 | branches: ["main"] 5 | pull_request: 6 | branches: ["main"] 7 | 8 | jobs: 9 | run-unit-tests: 10 | name: Run Unit Tests 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 14 | with: 15 | fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis 16 | - name: Set up Python 17 | uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 18 | with: 19 | python-version: "3.11" 20 | cache: "pipenv" 21 | - name: Install Pipenv 22 | run: pip install pipenv 23 | - name: Verify Pipfile.lock is in sync 24 | run: pipenv verify 25 | - name: Install dependencies 26 | run: pipenv install --dev 27 | - name: Run Unit Tests 28 | run: pipenv run tests 29 | - name: Show Coverage 30 | run: pipenv run tests_report 31 | - name: Upload Coverage to Codecov 32 | uses: codecov/codecov-action@ef609d6cb5624f374eed2390087f7ac0fc5f688a # v4.6.0 33 | with: 34 | token: ${{secrets.CODECOV_TOKEN}} 35 | -------------------------------------------------------------------------------- /.github/workflows/job-add-github-members-to-root-team-moj.yml: -------------------------------------------------------------------------------- 1 | name: 🤖 Add GitHub Members to Root Team (MoJ) 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: "0 8,10,12,14,16 * * 1-5" 6 | 7 | jobs: 8 | run-script: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 12 | - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 13 | with: 14 | python-version: "3.11" 15 | cache: "pipenv" 16 | - name: Install Pipenv 17 | run: | 18 | pip install pipenv 19 | pipenv install 20 | - run: pipenv run python3 -m bin.add_users_all_org_members_github_team 21 | env: 22 | ADMIN_GITHUB_TOKEN: ${{ secrets.OPS_ENG_GENERAL_ADMIN_BOT_PAT }} 23 | GITHUB_ORGANIZATION_NAME: ministryofjustice 24 | LOGGING_LEVEL: ${{ secrets.LOGGING_LEVEL }} 25 | - name: Report failure to Slack 26 | if: always() 27 | uses: ravsamhq/notify-slack-action@472601e839b758e36c455b5d3e5e1a217d4807bd # 2.5.0 28 | with: 29 | status: ${{ job.status }} 30 | notify_when: "failure" 31 | notification_title: "Failed GitHub Action Run" 32 | env: 33 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} 34 | -------------------------------------------------------------------------------- /.github/workflows/job-add-github-members-to-root-team-mojas.yml: -------------------------------------------------------------------------------- 1 | name: 🤖 Add GitHub Members to Root Team (MoJAS) 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: "0 8,10,12,14,16 * * 1-5" 6 | 7 | jobs: 8 | run-script: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 12 | - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 13 | with: 14 | python-version: "3.11" 15 | cache: "pipenv" 16 | - name: Install Pipenv 17 | run: | 18 | pip install pipenv 19 | pipenv install 20 | - run: pipenv run python3 -m bin.add_users_all_org_members_github_team 21 | env: 22 | ADMIN_GITHUB_TOKEN: ${{ secrets.OPS_ENG_GENERAL_ADMIN_BOT_PAT }} 23 | GITHUB_ORGANIZATION_NAME: moj-analytical-services 24 | LOGGING_LEVEL: ${{ secrets.LOGGING_LEVEL }} 25 | - name: Report failure to Slack 26 | if: always() 27 | uses: ravsamhq/notify-slack-action@472601e839b758e36c455b5d3e5e1a217d4807bd # 2.5.0 28 | with: 29 | status: ${{ job.status }} 30 | notify_when: "failure" 31 | notification_title: "Failed GitHub Action Run" 32 | env: 33 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} 34 | -------------------------------------------------------------------------------- /test/test_bin/test_check_mta_sts.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=C0411 2 | import unittest 3 | from services.s3_service import S3Service 4 | from unittest.mock import patch 5 | from moto import mock_aws 6 | from bin.check_mta_sts import main, check_mta_sts_domains 7 | 8 | class TestMTASTS(unittest.TestCase): 9 | @mock_aws 10 | def setUp(self): 11 | self.s3_service = S3Service("880656497252", "ministryofjustice") 12 | 13 | @mock_aws 14 | @patch.object(S3Service, "is_well_known_mta_sts_enforce") 15 | def test_check_mta_sts_with_enforce(self, mock_is_well_known_mta_sts_enforce): 16 | mock_is_well_known_mta_sts_enforce.return_value = True 17 | 18 | self.assertEqual(len(check_mta_sts_domains(self.s3_service)), 0) 19 | 20 | @mock_aws 21 | @patch.object(S3Service, "is_well_known_mta_sts_enforce") 22 | def test_check_mta_sts_without_enforce(self, mock_is_well_known_mta_sts_enforce): 23 | mock_is_well_known_mta_sts_enforce.side_effect = lambda domain: domain != "yjb.gov.uk" 24 | 25 | self.assertIn("yjb.gov.uk", check_mta_sts_domains(self.s3_service)) 26 | 27 | @patch('bin.check_mta_sts.check_mta_sts_domains') 28 | def test_main_function_with_enforce(self, mock_check_mta_sts_domains): 29 | main() 30 | mock_check_mta_sts_domains.assert_called_once() 31 | 32 | 33 | if __name__ == "__main__": 34 | unittest.main() 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/trivy-vulnerability-template.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Trivy Vulnerability Scan Report - {{ date | date('YYYY-MM-DD') }} 3 | about: Create an issue for vulnerabilities that need addressing 4 | title: Trivy Vulnerability Scan Report - {{ date | date('YYYY-MM-DD') }} 5 | labels: risk 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Trivy Vulnerability Scan Results 11 | 12 | A Trivy vulnerability scan has been conducted, revealing vulnerabilities across different severity levels. 13 | 14 | ### Action Required 15 | 16 | It's important that the team addresses these vulnerabilities as soon as possible to maintain the security integrity of our codebase. Please review the detailed scan results attached to the GitHub Security tab and initiate the necessary updates or patches. 17 | 18 | If a solution is not possible at this time, please raise it with the team for further discussion. 19 | 20 | ### Results 21 | 22 | Please refer to the [GitHub Security tab] (https://github.com/ministryofjustice/operations-engineering/security/code-scanning) for a detailed breakdown of each vulnerability found, including the package name, affected versions, and recommended fixes. 23 | 24 | ### Resources 25 | 26 | - [Trivy GitHub Action](https://github.com/aquasecurity/trivy-action) 27 | - [Understanding the GitHub Security Tab](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories) 28 | -------------------------------------------------------------------------------- /terraform/dsd/iam/deprecated/octodns_dns_write.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_user" "octodns_user" { 2 | name = "octodns-cicd-user" 3 | } 4 | 5 | resource "aws_iam_policy" "octodns_policy" { 6 | name = "OctoDNSPolicy" 7 | description = "Policy for OctoDNS to manage Route53" 8 | 9 | policy = jsonencode({ 10 | Version = "2012-10-17" 11 | Statement = [ 12 | { 13 | Effect = "Allow" 14 | Action = [ 15 | "route53:ChangeResourceRecordSets", 16 | "route53:CreateHostedZone", 17 | "route53:ListHealthChecks", 18 | "route53:ListHostedZones", 19 | "route53:ListHostedZonesByName", 20 | "route53:ListResourceRecordSets" 21 | ] 22 | Resource = "*" 23 | } 24 | ] 25 | }) 26 | } 27 | 28 | resource "aws_iam_user_policy_attachment" "octodns_user_policy_attachment" { 29 | user = aws_iam_user.octodns_user.name 30 | policy_arn = aws_iam_policy.octodns_policy.arn 31 | } 32 | 33 | resource "aws_iam_access_key" "octodns_access_key" { 34 | user = aws_iam_user.octodns_user.name 35 | } 36 | 37 | resource "github_actions_secret" "octodns_aws_access_key_id" { 38 | repository = "dns" 39 | secret_name = "OCTODNS_AWS_ACCESS_KEY_ID" 40 | plaintext_value = aws_iam_access_key.octodns_access_key.id 41 | } 42 | 43 | resource "github_actions_secret" "octodns_aws_secret_access_key" { 44 | repository = "dns" 45 | secret_name = "OCTODNS_AWS_SECRET_ACCESS_KEY" 46 | plaintext_value = aws_iam_access_key.octodns_access_key.secret 47 | } 48 | -------------------------------------------------------------------------------- /.mega-linter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configuration file for MegaLinter 3 | # See all available variables at https://megalinter.io/latest/config-file/ and in linters documentation 4 | 5 | APPLY_FIXES: all # all, none, or list of linter keys 6 | ENABLE_LINTERS: 7 | - ACTION_ACTIONLINT 8 | - BASH_SHELLCHECK 9 | - MARKDOWN_MARKDOWNLINT 10 | - DOCKERFILE_HADOLINT 11 | - REPOSITORY_GITLEAKS 12 | - YAML_PRETTIER 13 | - YAML_YAMLLINT 14 | - JSON_PRETTIER 15 | - PYTHON_PYLINT 16 | - PYTHON_FLAKE8 17 | - PYTHON_ISORT 18 | - TERRAFORM_TERRAFORM_FMT 19 | - TERRAFORM_TFLINT 20 | - TERRAFORM_TERRASCAN 21 | 22 | DISABLE_ERRORS: false 23 | SPELL_CSPELL_DISABLE_ERRORS: true 24 | MARKDOWN_MARKDOWN_LINK_CHECK_DISABLE_ERRORS: true 25 | SHOW_ELAPSED_TIME: true 26 | FILEIO_REPORTER: false 27 | PARALLEL: true 28 | GITHUB_STATUS_REPORTER: true 29 | GITHUB_COMMENT_REPORTER: true 30 | VALIDATE_ALL_CODEBASE: false 31 | LOG_LEVEL: INFO 32 | MARKDOWN_MARKDOWN_LINK_CHECK_ARGUMENTS: "-q" 33 | 34 | TERRAFORM_TFLINT_UNSECURED_ENV_VARIABLES: 35 | - GITHUB_TOKEN 36 | 37 | TERRAFORM_TERRASCAN_ARGUMENTS: "scan -d . -i terraform -v" 38 | 39 | # This threshold has been added as a temporary way to get around a particular error (unsupported block type). 40 | # Examples of this can be seen in the following issues, and appears to be a bug with Terrscan itself: 41 | # https://github.com/tenable/terrascan/issues/1615 42 | # https://github.com/tenable/terrascan/issues/1182 43 | # https://github.com/super-linter/super-linter/issues/3044 44 | 45 | TERRAFORM_TERRASCAN_DISABLE_ERRORS_IF_LESS_THAN: 2 46 | -------------------------------------------------------------------------------- /bin/check_mta_sts.py: -------------------------------------------------------------------------------- 1 | from services.s3_service import S3Service 2 | 3 | # List of MTA-STS domains 4 | domains = [ 5 | "ccrc.gov.uk", "cjit.gov.uk", "cshrcasework.justice.gov.uk", "devl.justice.gov.uk", 6 | "g.justice.gov.uk", "govfsl.com", "hmiprisons.gov.uk", "hmiprobation.gov.uk", 7 | "ima-citizensrights.org.uk", "imb.org.uk", "judicialappointments.gov.uk", 8 | "judicialconduct.gov.uk", "judicialombudsman.gov.uk", "judiciary.uk", "justice.gov.uk", 9 | "lawcommission.gov.uk", "newsletter.ima-citizensrights.org.uk", "obr.uk", "ospt.gov.uk", 10 | "ppo.gov.uk", "publicguardian.gov.uk", "sentencingcouncil.gov.uk", "sentencingcouncil.org.uk", 11 | "ukgovwales.gov.uk", "victimscommissioner.org.uk", "yjb.gov.uk", "yjbservicespp.yjb.gov.uk", 12 | "youthjusticepp.yjb.gov.uk" 13 | ] 14 | 15 | # Suffix for MTA-STS files 16 | SUFFIX = ".well-known/mta-sts.txt" 17 | 18 | 19 | def main(): 20 | s3_client = S3Service("880656497252", "ministryofjustice") 21 | failed_domains = check_mta_sts_domains(s3_client) 22 | 23 | if failed_domains: 24 | print(f"Domains failing MTA-STS enforcement:\n{', '.join(failed_domains)}") 25 | else: 26 | print("All domains enforce MTA-STS.") 27 | 28 | 29 | def check_mta_sts_domains(s3_client): 30 | failed_domains = [] 31 | 32 | for domain in domains: 33 | if not s3_client.is_well_known_mta_sts_enforce(domain): 34 | print(f"{domain} (No 'mode: enforce')") 35 | failed_domains.append(domain) 36 | 37 | return failed_domains 38 | 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /test/test_utils/test_environment.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | 4 | from utils.environment import EnvironmentVariables 5 | 6 | 7 | class TestEnvironmentVariables(unittest.TestCase): 8 | 9 | @patch.dict('os.environ', {'GH_MOJ_TOKEN': 'fake_moj_github_token', 'GH_MOJAS_TOKEN': 'fake_mojas_github_token', 'ADMIN_SLACK_TOKEN': 'fake_slack_token'}) 10 | def test_all_env_vars_set(self): 11 | env_vars = EnvironmentVariables( 12 | ['GH_MOJ_TOKEN', 'GH_MOJAS_TOKEN', 'ADMIN_SLACK_TOKEN']) 13 | self.assertEqual(env_vars.get('GH_MOJ_TOKEN'), 'fake_moj_github_token') 14 | self.assertEqual(env_vars.get('GH_MOJAS_TOKEN'), 'fake_mojas_github_token') 15 | self.assertEqual(env_vars.get('ADMIN_SLACK_TOKEN'), 'fake_slack_token') 16 | 17 | @patch.dict('os.environ', {}) 18 | def test_missing_env_vars(self): 19 | with self.assertRaises(EnvironmentError): 20 | EnvironmentVariables(['GH_MOJ_TOKEN', 'GH_MOJAS_TOKEN', 'ADMIN_SLACK_TOKEN']) 21 | 22 | @patch.dict('os.environ', {'GH_MOJ_TOKEN': 'fake_moj_github_token'}) 23 | def test_partial_env_vars_set(self): 24 | with self.assertRaises(EnvironmentError): 25 | EnvironmentVariables(['GH_MOJ_TOKEN', 'GH_MOJAS_TOKEN', 'ADMIN_SLACK_TOKEN']) 26 | 27 | @patch.dict('os.environ', {'GH_MOJ_TOKEN': 'fake_moj_github_token', 'GH_MOJAS_TOKEN': 'fake_mojas_github_token', 'ADMIN_SLACK_TOKEN': 'fake_slack_token'}) 28 | def test_extra_env_vars(self): 29 | env_vars = EnvironmentVariables( 30 | ['GH_MOJ_TOKEN', 'GH_MOJAS_TOKEN', 'ADMIN_SLACK_TOKEN', 'EXTRA_VAR']) 31 | self.assertEqual(env_vars.get('GH_MOJ_TOKEN'), 'fake_moj_github_token') 32 | self.assertEqual(env_vars.get('GH_MOJAS_TOKEN'), 'fake_mojas_github_token') 33 | self.assertEqual(env_vars.get('ADMIN_SLACK_TOKEN'), 'fake_slack_token') 34 | self.assertIsNone(env_vars.get('EXTRA_VAR')) 35 | 36 | 37 | if __name__ == '__main__': 38 | unittest.main() 39 | -------------------------------------------------------------------------------- /.github/workflows/job-identify-dormant-github-users-v2.yml: -------------------------------------------------------------------------------- 1 | name: ⚙️ Identify Dormant GitHub Users 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | days_since: 7 | type: choice 8 | description: "Select the number of days since to check for user dormancy." 9 | options: 10 | - "90" 11 | - "60" 12 | - "30" 13 | default: "90" 14 | schedule: 15 | - cron: "0 0 1 * *" 16 | 17 | jobs: 18 | identify_dormant_github_users: 19 | runs-on: ubuntu-latest 20 | permissions: 21 | id-token: write 22 | contents: read 23 | steps: 24 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 25 | 26 | - uses: ministryofjustice/github-actions/setup-aws-profile@721b0f273fc8356611cb18b3dfc02074d59217da # v18.2.1 27 | with: 28 | role-arn: ${{secrets.AWS_GITHUB_DORMANT_USERS_ARN}} 29 | profile-name: auth0_logs_profile 30 | 31 | - name: Setup Profile to Query CloudTrail in Modernisation Platform Account `operations-engineering-dev` 32 | uses: ministryofjustice/github-actions/setup-aws-profile@721b0f273fc8356611cb18b3dfc02074d59217da # v18.2.1 33 | with: 34 | role-arn: ${{secrets.AWS_MP_OPERATIONS_ENGINEERING_DEV_QUERY_CLOUDTRAIL_ROLE_ARN}} 35 | profile-name: operations_engineering_dev_query_cloudtrail 36 | 37 | - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 38 | with: 39 | python-version: "3.11" 40 | cache: "pipenv" 41 | 42 | - name: Install Pipenv 43 | run: | 44 | pip install pipenv 45 | pipenv install 46 | 47 | - run: pipenv run python3 -m bin.identify_dormant_github_users_v2 48 | env: 49 | GH_MOJ_TOKEN: ${{ secrets.GH_MOJ_DORMANT_USERS_READ }} 50 | GH_MOJAS_TOKEN: ${{ secrets.GH_MOJAS_DORMANT_USERS_READ }} 51 | ADMIN_SLACK_TOKEN: ${{ secrets.ADMIN_SEND_TO_SLACK }} 52 | DAYS_SINCE: ${{ inputs.days_since }} 53 | -------------------------------------------------------------------------------- /bin/add_users_all_org_members_github_team.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from services.github_service import GithubService 4 | 5 | MINISTRYOFJUSTICE_GITHUB_ORGANIZATION_NAME = "ministryofjustice" 6 | # Contains a base set of permissions for all users in MoJ 7 | MINISTRYOFJUSTICE_GITHUB_ORGANIZATION_BASE_TEAM_NAME = "all-org-members" 8 | 9 | MOJ_ANALYTICAL_SERVICES_GITHUB_ORGANIZATION_NAME = "moj-analytical-services" 10 | # Contains a base set of permissions for all users in MoJAS 11 | MOJ_ANALYTICAL_SERVICES_GITHUB_ORGANIZATION_BASE_TEAM_NAME = "everyone" 12 | 13 | 14 | def get_environment_variables() -> tuple[str, str]: 15 | github_token = os.getenv("ADMIN_GITHUB_TOKEN") 16 | if not github_token: 17 | raise ValueError( 18 | "The env variable ADMIN_GITHUB_TOKEN is empty or missing") 19 | 20 | github_organization_name = os.getenv("GITHUB_ORGANIZATION_NAME") 21 | if not github_organization_name: 22 | raise ValueError( 23 | "The env variable GITHUB_ORGANIZATION is empty or missing") 24 | 25 | return github_token, github_organization_name 26 | 27 | 28 | def get_config_for_organization(github_organization_name: str) -> tuple[str, str] | ValueError: 29 | if github_organization_name == MINISTRYOFJUSTICE_GITHUB_ORGANIZATION_NAME: 30 | return MINISTRYOFJUSTICE_GITHUB_ORGANIZATION_NAME, MINISTRYOFJUSTICE_GITHUB_ORGANIZATION_BASE_TEAM_NAME 31 | 32 | if github_organization_name == MOJ_ANALYTICAL_SERVICES_GITHUB_ORGANIZATION_NAME: 33 | return MOJ_ANALYTICAL_SERVICES_GITHUB_ORGANIZATION_NAME, MOJ_ANALYTICAL_SERVICES_GITHUB_ORGANIZATION_BASE_TEAM_NAME 34 | 35 | raise ValueError( 36 | f"Unsupported Github Organization Name [{github_organization_name}]") 37 | 38 | 39 | def main(): 40 | github_token, github_organization_name = get_environment_variables() 41 | organization_name, organization_team_name = get_config_for_organization( 42 | github_organization_name) 43 | GithubService(github_token, organization_name).add_all_users_to_team( 44 | organization_team_name) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /services/s3_service.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=wrong-import-order 2 | import os 3 | import boto3 4 | import json 5 | from botocore.exceptions import ClientError 6 | 7 | from json import JSONDecodeError 8 | 9 | 10 | class S3Service: 11 | def __init__(self, bucket_name: str, organisation_name: str) -> None: 12 | self.client = boto3.client("s3") 13 | self.bucket_name = bucket_name 14 | self.organisation_name = organisation_name 15 | 16 | def _download_file(self, object_name: str, file_path: str): 17 | self.client.download_file(self.bucket_name, object_name, file_path) 18 | if not os.path.isfile(file_path): 19 | raise ValueError( 20 | f"The {file_path} file did not download or is not in the expected location" 21 | ) 22 | 23 | def _upload_file(self, object_name: str, file_path: str): 24 | self.client.upload_file(file_path, self.bucket_name, object_name) 25 | 26 | def _delete_file(self, object_name: str): 27 | self.client.delete_object(Bucket=self.bucket_name, Key=object_name) 28 | 29 | def is_well_known_mta_sts_enforce(self, domain: str) -> bool: 30 | suffix = ".well-known/mta-sts.txt" 31 | bucket_name = f"880656497252.{domain}" 32 | try: 33 | response = self.client.get_object(Bucket=bucket_name, Key=suffix) 34 | sts_content = response['Body'].read().decode('utf-8') 35 | return any(line.startswith("mode: enforce") for line in sts_content.split('\n')) 36 | except ClientError: 37 | return False 38 | 39 | def get_json_file(self, object_name: str, file_path: str): 40 | 41 | try: 42 | with open(file_path, 'wb') as file: 43 | self.client.download_fileobj(self.bucket_name, object_name, file) 44 | with open(file_path, 'r', encoding="utf-8") as file: 45 | mappings = file.read() 46 | return json.loads(mappings) 47 | 48 | except FileNotFoundError as e: 49 | raise FileNotFoundError("Error downloading file") from e 50 | 51 | except JSONDecodeError as e: 52 | raise ValueError("File not in JSON Format") from e 53 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/operations-engineering-story.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: User Story 3 | about: Capture a user story from stakeholders. 4 | title: '' 5 | labels: user-story 6 | assignees: '' 7 | 8 | --- 9 | 10 | **User Need** 11 | 12 | **As a** (type of user) 13 | **I want** (some action) 14 | **so that** (some result) 15 | 16 | **Value** 17 | 18 | 19 | **Functional Requirements (What):** 20 | 21 | - [ ] 22 | - [ ] 23 | 24 | **Non-Functional Requirements (How):** 25 | 26 | - [ ] 27 | - [ ] 28 | 29 | 30 | **Acceptance Criteria:** 31 | 32 | - [ ] 33 | - [ ] 34 | 35 | 36 | **Assumptions (Optional):** 37 | 38 | - 39 | - 40 | 41 | **Risks and Mitigation (Optional):** 42 | 43 | - 44 | - 45 | 46 | **Notes:** 47 | 48 | 69 | -------------------------------------------------------------------------------- /.github/workflows/cicd-mega-linter.yml: -------------------------------------------------------------------------------- 1 | # MegaLinter GitHub Action configuration file 2 | # More info at https://megalinter.io 3 | name: ♻️ MegaLinter 4 | 5 | on: 6 | # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main 7 | # push: # Comment this line to trigger action only on pull-requests 8 | pull_request: 9 | branches: [main] 10 | workflow_dispatch: 11 | inputs: 12 | lint_entire_codebase: 13 | description: "Lint entire codebase?" 14 | required: false 15 | default: "false" 16 | 17 | concurrency: 18 | group: ${{ github.ref }}-${{ github.workflow }} 19 | cancel-in-progress: true 20 | 21 | jobs: 22 | megalinter: 23 | name: MegaLinter 24 | runs-on: ubuntu-latest 25 | permissions: 26 | # Give the default GITHUB_TOKEN write permission to commit and push, comment issues & post new PR 27 | # Remove the ones you do not need 28 | contents: write 29 | issues: write 30 | pull-requests: write 31 | steps: 32 | # Git Checkout 33 | - name: Checkout Code 34 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 35 | with: 36 | token: ${{ secrets.GITHUB_TOKEN }} 37 | fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances 38 | 39 | # MegaLinter 40 | - name: MegaLinter 41 | id: ml 42 | # DISABLE: COPYPASTE,SPELL # Uncomment to disable copy-paste and spell checks 43 | # You can override MegaLinter flavor used to have faster performances 44 | # More info at https://megalinter.io/latest/flavors/ 45 | uses: oxsecurity/megalinter/flavors/python@32c1b3827a334c80026c654f31ee1b4801ad8798 # v7.12.0 46 | env: 47 | # All available variables are described in documentation 48 | # https://megalinter.io/latest/configuration/ 49 | VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'workflow_dispatch' && inputs.lint_entire_codebase == 'true' }} # Validates all source when push on main, else just the git diff with main. Override with true if you always want to lint all sources 50 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 51 | # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOS 52 | # Upload MegaLinter artifacts 53 | - name: Archive production artifacts 54 | if: success() || failure() 55 | uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 56 | with: 57 | name: MegaLinter reports 58 | path: | 59 | megalinter-reports 60 | mega-linter.log 61 | -------------------------------------------------------------------------------- /test/test_bin/test_add_users_all_org_members_github_team.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from unittest.mock import MagicMock, patch 4 | 5 | from bin import add_users_all_org_members_github_team 6 | 7 | 8 | @patch("github.Github.__new__", new=MagicMock) 9 | @patch("gql.Client.__new__", new=MagicMock) 10 | @patch("gql.transport.aiohttp.AIOHTTPTransport.__new__", new=MagicMock) 11 | class TestAddUsersEveryoneGithubTeamMain(unittest.TestCase): 12 | 13 | @patch.dict(os.environ, {"ADMIN_GITHUB_TOKEN": "token", "GITHUB_ORGANIZATION_NAME": "ministryofjustice"}) 14 | def test_main_smoke_test(self): 15 | add_users_all_org_members_github_team.main() 16 | 17 | 18 | class TestAddUsersEveryoneGithubTeamGetEnvironmentVariables(unittest.TestCase): 19 | def test_raises_error_when_no_environment_variables_provided(self): 20 | self.assertRaises( 21 | ValueError, add_users_all_org_members_github_team.get_environment_variables) 22 | 23 | @patch.dict(os.environ, {"GITHUB_ORGANIZATION_NAME": "ministryofjustice"}) 24 | def test_raises_error_when_no_github_token(self): 25 | self.assertRaises( 26 | ValueError, add_users_all_org_members_github_team.get_environment_variables) 27 | 28 | @patch.dict(os.environ, {"ADMIN_GITHUB_TOKEN": "token"}) 29 | def test_raises_error_when_no_github_organization(self): 30 | self.assertRaises( 31 | ValueError, add_users_all_org_members_github_team.get_environment_variables) 32 | 33 | @patch.dict(os.environ, {"ADMIN_GITHUB_TOKEN": "token", "GITHUB_ORGANIZATION_NAME": "ministryofjustice"}) 34 | def test_returns_values(self): 35 | github_token, github_organization_name = add_users_all_org_members_github_team.get_environment_variables() 36 | self.assertEqual(github_token, "token") 37 | self.assertEqual(github_organization_name, "ministryofjustice") 38 | 39 | 40 | class TestAddUsersEveryoneGithubTeamGetConfigForOrganization(unittest.TestCase): 41 | def test_raises_error_when_unknown_github_organization(self): 42 | self.assertRaises( 43 | ValueError, add_users_all_org_members_github_team.get_config_for_organization, "unknown organization" 44 | ) 45 | 46 | def test_returns_values_ministryofjustice_config(self): 47 | organization_name, organization_team_name = add_users_all_org_members_github_team.get_config_for_organization( 48 | "ministryofjustice") 49 | self.assertEqual(organization_name, "ministryofjustice") 50 | self.assertEqual(organization_team_name, "all-org-members") 51 | 52 | def test_returns_values_moj_analytical_services_config(self): 53 | organization_name, organization_team_name = add_users_all_org_members_github_team.get_config_for_organization( 54 | "moj-analytical-services") 55 | self.assertEqual(organization_name, "moj-analytical-services") 56 | self.assertEqual(organization_team_name, "everyone") 57 | 58 | 59 | if __name__ == "__main__": 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /services/route53_service.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | import boto3 3 | 4 | 5 | @dataclass 6 | class RecordValueModel: 7 | value: str 8 | 9 | 10 | @dataclass 11 | class RecordSetModel: 12 | name: str 13 | type: str 14 | values: list[RecordValueModel] 15 | 16 | 17 | @dataclass 18 | class HostedZoneModel: 19 | name: str 20 | record_sets: list[RecordSetModel] 21 | 22 | 23 | class Route53Service: 24 | def __init__(self, profile: str) -> None: 25 | session = boto3.Session(profile_name=profile) 26 | self.client = session.client("route53") 27 | 28 | def __get_all_record_sets(self, zone_id: str) -> list[dict]: 29 | paginator = self.client.get_paginator("list_resource_record_sets") 30 | paginator_iterator = paginator.paginate(HostedZoneId=zone_id) 31 | 32 | record_sets = [] 33 | for page in paginator_iterator: 34 | record_sets.extend(page.get("ResourceRecordSets")) 35 | return record_sets 36 | 37 | def __get_all_hosted_zones(self) -> list[dict]: 38 | paginator = self.client.get_paginator("list_hosted_zones") 39 | paginator_iterator = paginator.paginate( 40 | PaginationConfig={ 41 | "MaxItems": 500, 42 | } 43 | ) 44 | 45 | record_sets = [] 46 | for page in paginator_iterator: 47 | record_sets.extend(page.get("HostedZones")) 48 | return record_sets 49 | 50 | def __get_hosted_zone_record_sets(self, zone_id: str) -> list[RecordSetModel]: 51 | all_record_sets = self.__get_all_record_sets(zone_id) 52 | 53 | record_sets: list[RecordSetModel] = [] 54 | for record_set in all_record_sets: 55 | record_set_name = record_set["Name"] 56 | record_set_type = record_set["Type"] 57 | 58 | record_values: list[RecordValueModel] = [] 59 | resource_records: list[dict[str, str]] 60 | try: 61 | resource_records = record_set["ResourceRecords"] 62 | except KeyError: 63 | resource_records = [] 64 | for record in resource_records: 65 | record_values.append(RecordValueModel(value=record["Value"])) 66 | 67 | record_sets.append( 68 | RecordSetModel( 69 | name=record_set_name, 70 | type=record_set_type, 71 | values=record_values, 72 | ) 73 | ) 74 | 75 | return record_sets 76 | 77 | def get_hosted_zones(self) -> list[HostedZoneModel]: 78 | all_hosted_zones = self.__get_all_hosted_zones() 79 | 80 | hosted_zones: list[HostedZoneModel] = [] 81 | for zone in all_hosted_zones: 82 | zone_name = zone["Name"] 83 | zone_id = zone["Id"] 84 | hosted_zones.append( 85 | HostedZoneModel( 86 | name=zone_name, 87 | record_sets=self.__get_hosted_zone_record_sets(zone_id), 88 | ) 89 | ) 90 | return hosted_zones 91 | -------------------------------------------------------------------------------- /test/test_services/test_s3_service.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=W0221, C0411 2 | import os 3 | import json 4 | import tempfile 5 | import unittest 6 | from botocore.exceptions import ClientError 7 | from unittest.mock import call, patch, mock_open 8 | from io import BytesIO 9 | from services.s3_service import S3Service 10 | 11 | 12 | class TestS3Service(unittest.TestCase): 13 | @patch("services.s3_service.boto3") 14 | def setUp(self, mock_boto3): 15 | self.mock_boto3 = mock_boto3 16 | self.s3_service = S3Service( 17 | "test-bucket", 18 | "some-org" 19 | ) 20 | self.s3_object_file = "s3_object_file.csv" 21 | self.the_json_file = "the_file.json" 22 | self.builtins = "builtins.open" 23 | self.fake_datetime = "2023-01-20T14:51:47.000+01:00" 24 | self.s3_json_file = "s3_object_file.json" 25 | 26 | def test_download_file_correctly(self): 27 | test_dir = tempfile.mkdtemp() 28 | file_path = os.path.join(test_dir, "the_file.csv") 29 | with open(file_path, "w", encoding="utf-8") as the_file: 30 | the_file.write("") 31 | 32 | self.s3_service._download_file(self.s3_object_file, file_path) 33 | self.mock_boto3.assert_has_calls( 34 | [call.client("s3"), call.client().download_file( 35 | "test-bucket", self.s3_object_file, file_path)] 36 | ) 37 | 38 | def test_download_file_raise_error(self): 39 | self.assertRaises( 40 | ValueError, self.s3_service._download_file, self.the_json_file, self.s3_json_file) 41 | 42 | def test_upload_file(self): 43 | self.s3_service._upload_file(self.the_json_file, self.s3_json_file) 44 | self.mock_boto3.assert_has_calls( 45 | [call.client("s3"), call.client().upload_file( 46 | self.s3_json_file, "test-bucket", self.the_json_file)] 47 | ) 48 | 49 | def test_delete_file(self): 50 | self.s3_service._delete_file(self.s3_json_file) 51 | self.mock_boto3.assert_has_calls( 52 | [call.client("s3"), call.client().delete_object( 53 | Bucket="test-bucket", Key=self.s3_json_file)] 54 | ) 55 | 56 | def test_is_well_known_mta_sts_enforce_enabled(self): 57 | self.s3_service.client.get_object.return_value = {'Body': BytesIO("mode: enforce".encode('utf-8'))} 58 | 59 | self.assertTrue(self.s3_service.is_well_known_mta_sts_enforce("example.com")) 60 | 61 | def test_is_well_known_mta_sts_enforce_disabled(self): 62 | self.s3_service.client.get_object.return_value = {'Body': BytesIO("mode: disabled".encode('utf-8'))} 63 | 64 | self.assertFalse(self.s3_service.is_well_known_mta_sts_enforce("example.com")) 65 | 66 | def test_is_well_known_mta_sts_enforce_no_such_key(self): 67 | self.s3_service.client.get_object.side_effect = ClientError( 68 | { 69 | 'Error': { 70 | 'Code': "test" 71 | } 72 | }, 73 | operation_name="test" 74 | ) 75 | 76 | self.assertFalse(self.s3_service.is_well_known_mta_sts_enforce("example.com")) 77 | 78 | def test_get_json_file_success(self): 79 | mock_content = '{"key": "value"}' 80 | with patch(self.builtins, mock_open(read_data=mock_content)) as mock_file: 81 | result = self.s3_service.get_json_file(self.the_json_file, self.s3_json_file) 82 | self.assertEqual(result, json.loads(mock_content)) 83 | self.mock_boto3.assert_has_calls([ 84 | call.client("s3"), 85 | call.client().download_fileobj(self.s3_service.bucket_name, self.the_json_file, mock_file()) 86 | ]) 87 | 88 | def test_get_json_file_not_found(self): 89 | with patch(self.builtins, mock_open()) as mock_file: 90 | mock_file.side_effect = FileNotFoundError 91 | with self.assertRaises(FileNotFoundError): 92 | self.s3_service.get_json_file(self.the_json_file, self.s3_json_file) 93 | 94 | def test_get_json_file_invalid_json(self): 95 | mock_content = 'Not a JSON' 96 | with patch(self.builtins, mock_open(read_data=mock_content)): 97 | with self.assertRaises(ValueError): 98 | self.s3_service.get_json_file(self.the_json_file, self.s3_json_file) 99 | 100 | 101 | if __name__ == "__main__": 102 | unittest.main() 103 | -------------------------------------------------------------------------------- /services/slack_service.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=C0411, R0917 2 | 3 | import logging 4 | from slack_sdk import WebClient 5 | from slack_sdk.errors import SlackApiError 6 | 7 | 8 | class SlackService: 9 | OPERATIONS_ENGINEERING_ALERTS_CHANNEL_ID = "C033QBE511V" 10 | OPERATIONS_ENGINEERING_TEAM_CHANNEL_ID = "CPVD6398C" 11 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 12 | OPERATION_ENGINEERING_REPOSITORY_URL = "https://github.com/ministryofjustice/operations-engineering" 13 | 14 | # Added to stop TypeError on instantiation. See https://github.com/python/cpython/blob/d2340ef25721b6a72d45d4508c672c4be38c67d3/Objects/typeobject.c#L4444 15 | def __new__(cls, *args, **kwargs): 16 | return super(SlackService, cls).__new__(cls) 17 | 18 | def __init__(self, slack_token: str) -> None: 19 | self.slack_client = WebClient(slack_token) 20 | 21 | def send_dormant_user_list(self, user_list, days_since: str): 22 | message = ( 23 | f"*Dormant User Report*\n\n" 24 | f"Here is a list of dormant GitHub users that have not been seen in Auth0 logs for {days_since} days:\n" 25 | f"{user_list}" 26 | ) 27 | blocks = self._create_block_with_message(message) 28 | self._send_alert_to_operations_engineering(blocks) 29 | 30 | def send_unknown_user_alert_to_operations_engineering(self, users: list): 31 | message = ( 32 | f"*Dormant Users Automation*\n\n" 33 | f"Remove these users from the Dormant Users allow list:\n" 34 | f"{users}" 35 | ) 36 | blocks = self._create_block_with_message(message) 37 | self._send_alert_to_operations_engineering(blocks) 38 | 39 | def send_remove_users_from_github_alert_to_operations_engineering( 40 | self, number_of_users: int, organisation_name: str 41 | ): 42 | message = ( 43 | f"*Dormant Users Automation*\n\n" 44 | f"Removed {number_of_users} users from the {organisation_name} GitHub Organisation.\n\n" 45 | f"See the GH Action for more info: {self.OPERATION_ENGINEERING_REPOSITORY_URL}" 46 | ) 47 | blocks = self._create_block_with_message(message) 48 | self._send_alert_to_operations_engineering(blocks) 49 | 50 | def send_undelivered_email_alert_to_operations_engineering( 51 | self, email_addresses: list, organisation_name: str 52 | ): 53 | message = ( 54 | f"*Dormant Users Automation*\n\n" 55 | f"Undelivered emails for {organisation_name} GitHub Organisation:\n" 56 | f"{email_addresses}\n\n" 57 | f"Remove these users manually." 58 | ) 59 | blocks = self._create_block_with_message(message) 60 | self._send_alert_to_operations_engineering(blocks) 61 | 62 | def send_remove_users_slack_message(self, number_of_users: int, organisation_name: str): 63 | message = f"*Dormant Users Automation*\nRemoved {number_of_users} users from the {organisation_name} GitHub Organisation.\nSee the GH Action for more info: {self.OPERATION_ENGINEERING_REPOSITORY_URL}" 64 | blocks = self._create_block_with_message(message) 65 | self._send_alert_to_operations_engineering(blocks) 66 | 67 | def send_unknown_users_slack_message(self, unknown_users: list): 68 | message = f"*Dormant Users Automation*\nRemove these users from the Dormant Users allow list:\n{unknown_users}" 69 | blocks = self._create_block_with_message(message) 70 | self._send_alert_to_operations_engineering(blocks) 71 | 72 | def send_undelivered_emails_slack_message(self, email_addresses: list, organisation_name: str): 73 | message = f"*Dormant Users Automation*\nUndelivered emails for {organisation_name} GitHub Organisation:\n{email_addresses}\nRemove these users manually" 74 | blocks = self._create_block_with_message(message) 75 | self._send_alert_to_operations_engineering(blocks) 76 | 77 | def _send_alert_to_operations_engineering(self, blocks: list[dict]): 78 | try: 79 | self.slack_client.chat_postMessage( 80 | channel=self.OPERATIONS_ENGINEERING_ALERTS_CHANNEL_ID, 81 | mrkdown=True, 82 | blocks=blocks 83 | ) 84 | except SlackApiError as e: 85 | logging.error("Slack API error: {%s}", e.response['error']) 86 | 87 | def _create_block_with_message(self, message, block_type="section"): 88 | return [ 89 | { 90 | "type": block_type, 91 | "text": { 92 | "type": "mrkdwn", 93 | "text": message 94 | } 95 | } 96 | ] 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Operations Engineering 2 | 3 | [![Ministry of Justice Repository Compliance Badge](https://github-community.service.justice.gov.uk/repository-standards/api/operations-engineering/badge)](https://github-community.service.justice.gov.uk/repository-standards/operations-engineering) 4 | 5 | This repository contains a collection of checks and reports developed and used by the Operations Engineering team at the Ministry of Justice. 6 | 7 | ## Our Vision 8 | 9 | The Operations Engineering team buy, build, and run tools to help build and operate software at scale. While we primarily use the [Cloud Platform](https://user-guide.cloud-platform.service.justice.gov.uk/) for developing and hosting our services, our goal is to create tools and standards that can be used across multiple platforms and hosting services throughout the organisation. 10 | 11 | ## What's in This Repo 12 | 13 | This mono repository includes code that performs various operations engineering tasks to streamline our workflow and maintain high operational standards. Here are some highlights: 14 | 15 | - **Repository Reports:** We generate reports that verify the adherence of the Ministry of Justice organisation's repositories to the high standards outlined in our [Repository Standards](https://operations-engineering-reports.cloud-platform.service.justice.gov.uk/). These reports help us maintain the quality of our code and streamline collaboration. 16 | 17 | - **Sentry Monitoring:** We monitor Sentry for over and under-utilisation, ensuring we leverage this error-tracking tool to its full potential. This helps in identifying and rectifying application errors more efficiently. 18 | 19 | - **Dormant User Detection:** Our code can detect inactive GitHub users and remove them from the organisation, keeping our workspace tidy and secure. 20 | 21 | - **Github Repository Terraform:** Our Github repositories are defined in Terraform. How to create a new repository is outlined in a [Runbook](https://runbooks.operations-engineering.service.justice.gov.uk/documentation/services/github/repository-terraform.html). 22 | 23 | These are just a few examples of this repository's useful tools and features. For more detailed information about each tool and feature and how they assist us in our operations, see the GitHub workflows in the `.github/workflows` directory for more information. 24 | 25 | ## Getting Started 26 | 27 | 1. **Clone the Repo:** `git clone https://github.com/ministryofjustice/operations-engineering.git` 28 | 2. **Install pre-commit:** `make local-setup` 29 | 3. **Navigate to the Repo:** `cd operations-engineering` 30 | 4. **Install Dependencies:** `pipenv install --dev` 31 | 5. **Run a script:** `pipenv run python -m bin.identify_dormant_github_users` 32 | 33 | ## Pipenv 34 | 35 | ### Basics 36 | 37 | ```bash 38 | # Install pipenv via brew 39 | brew install pipenv 40 | # or via pip 41 | python3 -m pip install pipenv 42 | 43 | # Install all dependencies, including development dependencies 44 | pipenv install --dev 45 | 46 | # Run a script within the created virtual environment 47 | pipenv run tests 48 | # or as a command 49 | pipenv run coverage run python -m unittest 50 | 51 | # Get location of virtual environment (location may be needed to point your IDE to the right interpreter) 52 | pipenv --venv 53 | 54 | # Check for vulnerable packages 55 | pipenv check 56 | 57 | # Additional information on pipenv functionality 58 | pipenv --help 59 | ``` 60 | 61 | ## Naming Standards For Workflow Files 62 | 63 | To aid navigation, standardisation and deprecation of workflows - we have opted to follow a simple naming convention for the different types of workflows that are contained within the repository. 64 | 65 | For this, we use a prefix in the workflow filename - to ensure similar workflows are next to each other in most local development environments and an emoji in the workflow name - to ensure it's easily findable in the GitHub Actions UI. 66 | 67 | Please ensure any new workflow files are prefixed with one of the below standards. 68 | 69 | ### `cicd-` 70 | 71 | For any workflow that is purely related to Continuous Integration and Continuous Deployment, i.e. checks on PRs, deploying to environments etc. 72 | 73 | Prefix the workflow name with: ♻️ 74 | 75 | ### `job-` 76 | 77 | For any workflow related to executing code that should be run periodically (whether automated or manual). This mainly relates to business processes that are automated to some degree. 78 | 79 | If the job is completely automated (i.e. runs on a defined schedule), prefix the workflow name with: 🤖 80 | 81 | If the job needs to be triggered manually, prefix the workflow name with: 🧑‍🔧 82 | 83 | #### `experiment-` 84 | 85 | For any workflow that is currently under testing, potentially for a proof-of-concept and isn't essential to any current process. 86 | 87 | Prefix the workflow name with: 🧪 88 | 89 | ## Support 90 | 91 | If you have any questions or need help with this repository, please contact us on the #ask-operations-engineering slack channel. 92 | 93 | ## License 94 | 95 | This project is licensed under the [MIT License](/LICENSE.md). 96 | -------------------------------------------------------------------------------- /test/test_services/test_slack_service.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=W0221, C0411 2 | 3 | import unittest 4 | from unittest.mock import MagicMock, patch 5 | 6 | from services.slack_service import SlackService 7 | 8 | @patch("slack_sdk.WebClient.__new__") 9 | class TestSlackServiceInit(unittest.TestCase): 10 | 11 | def test_sets_up_class(self, mock_slack_client: MagicMock): 12 | mock_slack_client.return_value = "test_mock" 13 | slack_service = SlackService("") 14 | self.assertEqual("test_mock", 15 | slack_service.slack_client) 16 | 17 | 18 | class TestSendMessageToPlainTextChannelName(unittest.TestCase): 19 | 20 | @patch("slack_sdk.WebClient.__new__") 21 | def setUp(self, mock_web_client): 22 | self.channel_name = 'test_channel' 23 | self.message = 'test message' 24 | self.channel_id = 'test_channel_id' 25 | self.response_metadata = {'next_cursor': ''} 26 | self.channel = {'name': self.channel_name, 'id': self.channel_id} 27 | self.response = {'ok': True} 28 | self.slack_client = MagicMock() 29 | self.mock_web_client = mock_web_client 30 | self.mock_web_client.return_value = self.slack_client 31 | self.slack_service = SlackService("") 32 | self.slack_client.conversations_list.return_value = { 33 | 'channels': [self.channel], 'response_metadata': self.response_metadata} 34 | self.slack_service.slack_client = self.slack_client 35 | 36 | 37 | @patch("slack_sdk.WebClient.__new__") 38 | class SendUnknownUserAlertToOperationsEngineering(unittest.TestCase): 39 | 40 | def test_downstream_services_called(self, mock_slack_client: MagicMock): 41 | users = ["some-user1", "some-user2", "some-user3"] 42 | SlackService( 43 | "").send_unknown_user_alert_to_operations_engineering(users) 44 | mock_slack_client.return_value.chat_postMessage.assert_called_with( 45 | channel="C033QBE511V", 46 | mrkdown=True, 47 | blocks=[ 48 | { 49 | "type": "section", 50 | "text": { 51 | "type": "mrkdwn", 52 | "text": '*Dormant Users Automation*\n\nRemove these users from the Dormant Users allow list:\n[\'some-user1\', \'some-user2\', \'some-user3\']' 53 | } 54 | } 55 | ] 56 | ) 57 | 58 | 59 | @patch("slack_sdk.WebClient.__new__") 60 | class SendRemoveUsersFromGithubAlertToOperationsEngineering(unittest.TestCase): 61 | 62 | def test_downstream_services_called(self, mock_slack_client: MagicMock): 63 | SlackService("").send_remove_users_from_github_alert_to_operations_engineering( 64 | 3, "some-org") 65 | mock_slack_client.return_value.chat_postMessage.assert_called_with( 66 | channel="C033QBE511V", 67 | mrkdown=True, 68 | blocks=[ 69 | { 70 | "type": "section", 71 | "text": { 72 | "type": "mrkdwn", 73 | "text": '*Dormant Users Automation*\n\nRemoved 3 users from the some-org GitHub Organisation.\n\nSee the GH Action for more info: https://github.com/ministryofjustice/operations-engineering' 74 | } 75 | } 76 | ] 77 | ) 78 | 79 | 80 | @patch("slack_sdk.WebClient.__new__") 81 | class SendUndeliveredEmailAlertToOperationsEngineering(unittest.TestCase): 82 | 83 | def test_downstream_services_called(self, mock_slack_client: MagicMock): 84 | email_address = ["some-user1@domain.com", 85 | "some-user2@domain.com", "some-user3@domain.com"] 86 | SlackService("").send_undelivered_email_alert_to_operations_engineering( 87 | email_address, "some-org") 88 | mock_slack_client.return_value.chat_postMessage.assert_called_with( 89 | channel="C033QBE511V", 90 | mrkdown=True, 91 | blocks=[ 92 | { 93 | "type": "section", 94 | "text": { 95 | "type": "mrkdwn", 96 | "text": '*Dormant Users Automation*\n\nUndelivered emails for some-org GitHub Organisation:\n[\'some-user1@domain.com\', \'some-user2@domain.com\', \'some-user3@domain.com\']\n\nRemove these users manually.' 97 | } 98 | } 99 | ] 100 | ) 101 | 102 | 103 | class TestSlackService(unittest.TestCase): 104 | 105 | @patch("slack_sdk.WebClient.__new__") 106 | def setUp(self, mock_slack_client): 107 | self.message = "some-message" 108 | self.blocks = [ 109 | { 110 | "type": "section", 111 | "text": { 112 | "type": "mrkdwn", 113 | "text": self.message 114 | } 115 | } 116 | ] 117 | self.mock_slack_client = mock_slack_client 118 | self.slack_service = SlackService("") 119 | 120 | def test_create_block_with_message(self): 121 | self.assertEqual( 122 | self.blocks, self.slack_service._create_block_with_message(self.message)) 123 | 124 | def test_send_alert_to_operations_engineering(self): 125 | self.slack_service._send_alert_to_operations_engineering(self.blocks) 126 | self.mock_slack_client.return_value.chat_postMessage.assert_called_with( 127 | channel="C033QBE511V", 128 | mrkdown=True, 129 | blocks=self.blocks 130 | ) 131 | 132 | def test_send_dormant_user_list(self): 133 | user_list = "Test user list" 134 | expected_message = ( 135 | "*Dormant User Report*\n\n" 136 | "Here is a list of dormant GitHub users that have not been seen in Auth0 logs for 13 days:\n" 137 | f"{user_list}" 138 | ) 139 | blocks = self.slack_service._create_block_with_message(expected_message) 140 | self.slack_service.send_dormant_user_list(user_list, str(13)) 141 | self.mock_slack_client.return_value.chat_postMessage.assert_called_once_with( 142 | channel="C033QBE511V", mrkdown=True, blocks=blocks 143 | ) 144 | 145 | def test_send_unknown_user_alert_to_operations_engineering(self): 146 | users = ["user1", "user2"] 147 | expected_message = ( 148 | "*Dormant Users Automation*\n\n" 149 | "Remove these users from the Dormant Users allow list:\n" 150 | f"{users}" 151 | ) 152 | blocks = self.slack_service._create_block_with_message(expected_message) 153 | self.slack_service.send_unknown_user_alert_to_operations_engineering(users) 154 | self.mock_slack_client.return_value.chat_postMessage.assert_called_once_with( 155 | channel="C033QBE511V", mrkdown=True, blocks=blocks 156 | ) 157 | 158 | def test_send_remove_users_from_github_alert(self): 159 | number_of_users = 3 160 | organisation_name = "Test Org" 161 | expected_message = ( 162 | "*Dormant Users Automation*\n\n" 163 | f"Removed {number_of_users} users from the {organisation_name} GitHub Organisation.\n\n" 164 | f"See the GH Action for more info: https://github.com/ministryofjustice/operations-engineering" 165 | ) 166 | blocks = self.slack_service._create_block_with_message(expected_message) 167 | self.slack_service.send_remove_users_from_github_alert_to_operations_engineering(number_of_users, organisation_name) 168 | self.mock_slack_client.return_value.chat_postMessage.assert_called_once_with( 169 | channel="C033QBE511V", mrkdown=True, blocks=blocks 170 | ) 171 | 172 | def test_send_unknown_users_slack_message(self): 173 | self.slack_service.send_unknown_users_slack_message( 174 | ["some-user1", "some-user2", "some-user3"]) 175 | self.mock_slack_client.return_value.chat_postMessage.assert_called_with( 176 | channel="C033QBE511V", 177 | mrkdown=True, 178 | blocks=[ 179 | { 180 | "type": "section", 181 | "text": { 182 | "type": "mrkdwn", 183 | "text": '*Dormant Users Automation*\nRemove these users from the Dormant Users allow list:\n[\'some-user1\', \'some-user2\', \'some-user3\']' 184 | } 185 | } 186 | ] 187 | ) 188 | 189 | def test_send_remove_users_slack_message(self): 190 | self.slack_service.send_remove_users_slack_message( 191 | 3, "some-org") 192 | self.mock_slack_client.return_value.chat_postMessage.assert_called_with( 193 | channel="C033QBE511V", 194 | mrkdown=True, 195 | blocks=[ 196 | { 197 | "type": "section", 198 | "text": { 199 | "type": "mrkdwn", 200 | "text": '*Dormant Users Automation*\nRemoved 3 users from the some-org GitHub Organisation.\nSee the GH Action for more info: https://github.com/ministryofjustice/operations-engineering' 201 | } 202 | } 203 | ] 204 | ) 205 | 206 | def test_send_undelivered_emails_slack_message(self): 207 | email_address = ["some-user1@domain.com", 208 | "some-user2@domain.com", 209 | "some-user3@domain.com" 210 | ] 211 | self.slack_service.send_undelivered_emails_slack_message( 212 | email_address, "some-org") 213 | self.mock_slack_client.return_value.chat_postMessage.assert_called_with( 214 | channel="C033QBE511V", 215 | mrkdown=True, 216 | blocks=[ 217 | { 218 | "type": "section", 219 | "text": { 220 | "type": "mrkdwn", 221 | "text": '*Dormant Users Automation*\nUndelivered emails for some-org GitHub Organisation:\n[\'some-user1@domain.com\', \'some-user2@domain.com\', \'some-user3@domain.com\']\nRemove these users manually' 222 | } 223 | } 224 | ] 225 | ) 226 | 227 | 228 | if __name__ == "__main__": 229 | unittest.main() 230 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MAIN] 2 | 3 | # Analyse import fallback blocks. This can be used to support both Python 2 and 4 | # 3 compatible code, which means that the block might have code that exists 5 | # only in one or another interpreter, leading to false positives when analysed. 6 | analyse-fallback-blocks=no 7 | 8 | # Load and enable all available extensions. Use --list-extensions to see a list 9 | # all available extensions. 10 | #enable-all-extensions= 11 | 12 | # In error mode, messages with a category besides ERROR or FATAL are 13 | # suppressed, and no reports are done by default. Error mode is compatible with 14 | # disabling specific errors. 15 | #errors-only= 16 | 17 | # Always return a 0 (non-error) status code, even if lint errors are found. 18 | # This is primarily useful in continuous integration scripts. 19 | #exit-zero= 20 | 21 | # A comma-separated list of package or module names from where C extensions may 22 | # be loaded. Extensions are loading into the active Python interpreter and may 23 | # run arbitrary code. 24 | extension-pkg-allow-list= 25 | 26 | # A comma-separated list of package or module names from where C extensions may 27 | # be loaded. Extensions are loading into the active Python interpreter and may 28 | # run arbitrary code. (This is an alternative name to extension-pkg-allow-list 29 | # for backward compatibility.) 30 | extension-pkg-whitelist= 31 | 32 | # Return non-zero exit code if any of these messages/categories are detected, 33 | # even if score is above --fail-under value. Syntax same as enable. Messages 34 | # specified are enabled, while categories only check already-enabled messages. 35 | fail-on= 36 | 37 | # Specify a score threshold under which the program will exit with error. 38 | fail-under=10 39 | 40 | # Interpret the stdin as a python script, whose filename needs to be passed as 41 | # the module_or_package argument. 42 | #from-stdin= 43 | 44 | # Files or directories to be skipped. They should be base names, not paths. 45 | ignore=CVS 46 | 47 | # Add files or directories matching the regular expressions patterns to the 48 | # ignore-list. The regex matches against paths and can be in Posix or Windows 49 | # format. Because '\' represents the directory delimiter on Windows systems, it 50 | # can't be used as an escape character. 51 | ignore-paths= 52 | 53 | # Files or directories matching the regular expression patterns are skipped. 54 | # The regex matches against base names, not paths. The default value ignores 55 | # Emacs file locks 56 | ignore-patterns=^\.# 57 | 58 | # List of module names for which member attributes should not be checked 59 | # (useful for modules/projects where namespaces are manipulated during runtime 60 | # and thus existing member attributes cannot be deduced by static analysis). It 61 | # supports qualified module names, as well as Unix pattern matching. 62 | ignored-modules= 63 | 64 | # Python code to execute, usually for sys.path manipulation such as 65 | # pygtk.require(). 66 | #init-hook= 67 | 68 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 69 | # number of processors available to use, and will cap the count on Windows to 70 | # avoid hangs. 71 | jobs=1 72 | 73 | # Control the amount of potential inferred values when inferring a single 74 | # object. This can help the performance when dealing with large functions or 75 | # complex, nested conditions. 76 | limit-inference-results=100 77 | 78 | # List of plugins (as comma separated values of python module names) to load, 79 | # usually to register additional checkers. 80 | load-plugins= 81 | 82 | # Pickle collected data for later comparisons. 83 | persistent=yes 84 | 85 | # Minimum Python version to use for version dependent checks. Will default to 86 | # the version used to run pylint. 87 | py-version=3.11 88 | 89 | # Discover python modules and packages in the file system subtree. 90 | recursive=no 91 | 92 | # When enabled, pylint would attempt to guess common misconfiguration and emit 93 | # user-friendly hints instead of false-positive error messages. 94 | suggestion-mode=yes 95 | 96 | # Allow loading of arbitrary C extensions. Extensions are imported into the 97 | # active Python interpreter and may run arbitrary code. 98 | unsafe-load-any-extension=no 99 | 100 | # In verbose mode, extra non-checker-related info will be displayed. 101 | #verbose= 102 | 103 | 104 | [BASIC] 105 | 106 | # Naming style matching correct argument names. 107 | argument-naming-style=snake_case 108 | 109 | # Regular expression matching correct argument names. Overrides argument- 110 | # naming-style. If left empty, argument names will be checked with the set 111 | # naming style. 112 | #argument-rgx= 113 | 114 | # Naming style matching correct attribute names. 115 | attr-naming-style=snake_case 116 | 117 | # Regular expression matching correct attribute names. Overrides attr-naming- 118 | # style. If left empty, attribute names will be checked with the set naming 119 | # style. 120 | #attr-rgx= 121 | 122 | # Bad variable names which should always be refused, separated by a comma. 123 | bad-names=foo, 124 | bar, 125 | baz, 126 | toto, 127 | tutu, 128 | tata 129 | 130 | # Bad variable names regexes, separated by a comma. If names match any regex, 131 | # they will always be refused 132 | bad-names-rgxs= 133 | 134 | # Naming style matching correct class attribute names. 135 | class-attribute-naming-style=any 136 | 137 | # Regular expression matching correct class attribute names. Overrides class- 138 | # attribute-naming-style. If left empty, class attribute names will be checked 139 | # with the set naming style. 140 | #class-attribute-rgx= 141 | 142 | # Naming style matching correct class constant names. 143 | class-const-naming-style=UPPER_CASE 144 | 145 | # Regular expression matching correct class constant names. Overrides class- 146 | # const-naming-style. If left empty, class constant names will be checked with 147 | # the set naming style. 148 | #class-const-rgx= 149 | 150 | # Naming style matching correct class names. 151 | class-naming-style=PascalCase 152 | 153 | # Regular expression matching correct class names. Overrides class-naming- 154 | # style. If left empty, class names will be checked with the set naming style. 155 | #class-rgx= 156 | 157 | # Naming style matching correct constant names. 158 | const-naming-style=UPPER_CASE 159 | 160 | # Regular expression matching correct constant names. Overrides const-naming- 161 | # style. If left empty, constant names will be checked with the set naming 162 | # style. 163 | #const-rgx= 164 | 165 | # Minimum line length for functions/classes that require docstrings, shorter 166 | # ones are exempt. 167 | docstring-min-length=-1 168 | 169 | # Naming style matching correct function names. 170 | function-naming-style=snake_case 171 | 172 | # Regular expression matching correct function names. Overrides function- 173 | # naming-style. If left empty, function names will be checked with the set 174 | # naming style. 175 | #function-rgx= 176 | 177 | # Good variable names which should always be accepted, separated by a comma. 178 | good-names=i, 179 | j, 180 | k, 181 | ex, 182 | Run, 183 | _ 184 | 185 | # Good variable names regexes, separated by a comma. If names match any regex, 186 | # they will always be accepted 187 | good-names-rgxs= 188 | 189 | # Include a hint for the correct naming format with invalid-name. 190 | include-naming-hint=no 191 | 192 | # Naming style matching correct inline iteration names. 193 | inlinevar-naming-style=any 194 | 195 | # Regular expression matching correct inline iteration names. Overrides 196 | # inlinevar-naming-style. If left empty, inline iteration names will be checked 197 | # with the set naming style. 198 | #inlinevar-rgx= 199 | 200 | # Naming style matching correct method names. 201 | method-naming-style=snake_case 202 | 203 | # Regular expression matching correct method names. Overrides method-naming- 204 | # style. If left empty, method names will be checked with the set naming style. 205 | #method-rgx= 206 | 207 | # Naming style matching correct module names. 208 | module-naming-style=snake_case 209 | 210 | # Regular expression matching correct module names. Overrides module-naming- 211 | # style. If left empty, module names will be checked with the set naming style. 212 | #module-rgx= 213 | 214 | # Colon-delimited sets of names that determine each other's naming style when 215 | # the name regexes allow several styles. 216 | name-group= 217 | 218 | # Regular expression which should only match function or class names that do 219 | # not require a docstring. 220 | no-docstring-rgx=^_ 221 | 222 | # List of decorators that produce properties, such as abc.abstractproperty. Add 223 | # to this list to register other decorators that produce valid properties. 224 | # These decorators are taken in consideration only for invalid-name. 225 | property-classes=abc.abstractproperty 226 | 227 | # Regular expression matching correct type variable names. If left empty, type 228 | # variable names will be checked with the set naming style. 229 | #typevar-rgx= 230 | 231 | # Naming style matching correct variable names. 232 | variable-naming-style=snake_case 233 | 234 | # Regular expression matching correct variable names. Overrides variable- 235 | # naming-style. If left empty, variable names will be checked with the set 236 | # naming style. 237 | #variable-rgx= 238 | 239 | 240 | [CLASSES] 241 | 242 | # Warn about protected attribute access inside special methods 243 | check-protected-access-in-special-methods=no 244 | 245 | # List of method names used to declare (i.e. assign) instance attributes. 246 | defining-attr-methods=__init__, 247 | __new__, 248 | setUp, 249 | __post_init__ 250 | 251 | # List of member names, which should be excluded from the protected access 252 | # warning. 253 | exclude-protected=_asdict, 254 | _fields, 255 | _replace, 256 | _source, 257 | _make 258 | 259 | # List of valid names for the first argument in a class method. 260 | valid-classmethod-first-arg=cls 261 | 262 | # List of valid names for the first argument in a metaclass class method. 263 | valid-metaclass-classmethod-first-arg=cls 264 | 265 | 266 | [DESIGN] 267 | 268 | # List of regular expressions of class ancestor names to ignore when counting 269 | # public methods (see R0903) 270 | exclude-too-few-public-methods= 271 | 272 | # List of qualified class names to ignore when counting class parents (see 273 | # R0901) 274 | ignored-parents= 275 | 276 | # Maximum number of arguments for function / method. 277 | max-args=10 278 | 279 | # Maximum number of attributes for a class (see R0902). 280 | max-attributes=10 281 | 282 | # Maximum number of boolean expressions in an if statement (see R0916). 283 | max-bool-expr=5 284 | 285 | # Maximum number of branch for function / method body. 286 | max-branches=12 287 | 288 | # Maximum number of locals for function / method body. 289 | max-locals=20 290 | 291 | # Maximum number of parents for a class (see R0901). 292 | max-parents=7 293 | 294 | # Maximum number of public methods for a class (see R0904). 295 | max-public-methods=20 296 | 297 | # Maximum number of return / yield for function / method body. 298 | max-returns=6 299 | 300 | # Maximum number of statements in function / method body. 301 | max-statements=50 302 | 303 | # Minimum number of public methods for a class (see R0903). 304 | min-public-methods=2 305 | 306 | 307 | [EXCEPTIONS] 308 | 309 | # Exceptions that will emit a warning when caught. 310 | overgeneral-exceptions=builtins.BaseException, 311 | builtins.Exception 312 | 313 | 314 | [FORMAT] 315 | 316 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 317 | expected-line-ending-format= 318 | 319 | # Regexp for a line that is allowed to be longer than the limit. 320 | ignore-long-lines=^\s*(# )??$ 321 | 322 | # Number of spaces of indent required inside a hanging or continued line. 323 | indent-after-paren=4 324 | 325 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 326 | # tab). 327 | indent-string=' ' 328 | 329 | # Maximum number of characters on a single line. 330 | max-line-length=100 331 | 332 | # Maximum number of lines in a module. 333 | max-module-lines=1000 334 | 335 | # Allow the body of a class to be on the same line as the declaration if body 336 | # contains single statement. 337 | single-line-class-stmt=no 338 | 339 | # Allow the body of an if to be on the same line as the test if there is no 340 | # else. 341 | single-line-if-stmt=no 342 | 343 | 344 | [IMPORTS] 345 | 346 | # List of modules that can be imported at any level, not just the top level 347 | # one. 348 | allow-any-import-level= 349 | 350 | # Allow wildcard imports from modules that define __all__. 351 | allow-wildcard-with-all=no 352 | 353 | # Deprecated modules which should not be used, separated by a comma. 354 | deprecated-modules= 355 | 356 | # Output a graph (.gv or any supported image format) of external dependencies 357 | # to the given file (report RP0402 must not be disabled). 358 | ext-import-graph= 359 | 360 | # Output a graph (.gv or any supported image format) of all (i.e. internal and 361 | # external) dependencies to the given file (report RP0402 must not be 362 | # disabled). 363 | import-graph= 364 | 365 | # Output a graph (.gv or any supported image format) of internal dependencies 366 | # to the given file (report RP0402 must not be disabled). 367 | int-import-graph= 368 | 369 | # Force import order to recognize a module as part of the standard 370 | # compatibility libraries. 371 | known-standard-library= 372 | 373 | # Force import order to recognize a module as part of a third party library. 374 | known-third-party=enchant 375 | 376 | # Couples of modules and preferred modules, separated by a comma. 377 | preferred-modules= 378 | 379 | 380 | [LOGGING] 381 | 382 | # The type of string formatting that logging methods do. `old` means using % 383 | # formatting, `new` is for `{}` formatting. 384 | logging-format-style=old 385 | 386 | # Logging modules to check that the string format arguments are in logging 387 | # function parameter format. 388 | logging-modules=logging 389 | 390 | 391 | [MESSAGES CONTROL] 392 | 393 | # Only show warnings with the listed confidence levels. Leave empty to show 394 | # all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, 395 | # UNDEFINED. 396 | confidence=HIGH, 397 | CONTROL_FLOW, 398 | INFERENCE, 399 | INFERENCE_FAILURE, 400 | UNDEFINED 401 | 402 | # Disable the message, report, category or checker with the given id(s). You 403 | # can either give multiple identifiers separated by comma (,) or put this 404 | # option multiple times (only on the command line, not in the configuration 405 | # file where it should appear only once). You can also use "--disable=all" to 406 | # disable everything first and then re-enable specific checks. For example, if 407 | # you want to run only the similarities checker, you can use "--disable=all 408 | # --enable=similarities". If you want to run only the classes checker, but have 409 | # no Warning level messages displayed, use "--disable=all --enable=classes 410 | # --disable=W". 411 | disable=raw-checker-failed, 412 | bad-inline-option, 413 | locally-disabled, 414 | file-ignored, 415 | suppressed-message, 416 | useless-suppression, 417 | deprecated-pragma, 418 | use-symbolic-message-instead, 419 | missing-class-docstring, 420 | missing-module-docstring, 421 | missing-function-docstring, 422 | line-too-long, 423 | C0302, # Too many lines in module 424 | C2801, # Unnecessarily calls dunder method 425 | E0401, # Failed to import package 426 | R0801, # Similar lines in two files 427 | R0902, # Too many instance attributes 428 | R0903, # Too few public methods 429 | R0904, # Too many public methods 430 | R0917, # Too many positional arguments 431 | W0212 # Access to a protected member (we do this for testing) 432 | 433 | 434 | # Enable the message, report, category or checker with the given id(s). You can 435 | # either give multiple identifier separated by comma (,) or put this option 436 | # multiple time (only on the command line, not in the configuration file where 437 | # it should appear only once). See also the "--disable" option for examples. 438 | enable=c-extension-no-member 439 | 440 | 441 | [METHOD_ARGS] 442 | 443 | # List of qualified names (i.e., library.method) which require a timeout 444 | # parameter e.g. 'requests.api.get,requests.api.post' 445 | timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request 446 | 447 | 448 | [MISCELLANEOUS] 449 | 450 | # List of note tags to take in consideration, separated by a comma. 451 | notes=FIXME, 452 | XXX, 453 | TODO 454 | 455 | # Regular expression of note tags to take in consideration. 456 | notes-rgx= 457 | 458 | 459 | [REFACTORING] 460 | 461 | # Maximum number of nested blocks for function / method body 462 | max-nested-blocks=5 463 | 464 | # Complete name of functions that never returns. When checking for 465 | # inconsistent-return-statements if a never returning function is called then 466 | # it will be considered as an explicit return statement and no message will be 467 | # printed. 468 | never-returning-functions=sys.exit,argparse.parse_error 469 | 470 | 471 | [REPORTS] 472 | 473 | # Python expression which should return a score less than or equal to 10. You 474 | # have access to the variables 'fatal', 'error', 'warning', 'refactor', 475 | # 'convention', and 'info' which contain the number of messages in each 476 | # category, as well as 'statement' which is the total number of statements 477 | # analyzed. This score is used by the global evaluation report (RP0004). 478 | evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) 479 | 480 | # Template used to display messages. This is a python new-style format string 481 | # used to format the message information. See doc for all details. 482 | msg-template= 483 | 484 | # Set the output format. Available formats are text, parseable, colorized, json 485 | # and msvs (visual studio). You can also give a reporter class, e.g. 486 | # mypackage.mymodule.MyReporterClass. 487 | #output-format= 488 | 489 | # Tells whether to display a full report or only the messages. 490 | reports=no 491 | 492 | # Activate the evaluation score. 493 | score=yes 494 | 495 | 496 | [SIMILARITIES] 497 | 498 | # Comments are removed from the similarity computation 499 | ignore-comments=yes 500 | 501 | # Docstrings are removed from the similarity computation 502 | ignore-docstrings=yes 503 | 504 | # Imports are removed from the similarity computation 505 | ignore-imports=yes 506 | 507 | # Signatures are removed from the similarity computation 508 | ignore-signatures=yes 509 | 510 | # Minimum lines number of a similarity. 511 | min-similarity-lines=4 512 | 513 | 514 | [SPELLING] 515 | 516 | # Limits count of emitted suggestions for spelling mistakes. 517 | max-spelling-suggestions=4 518 | 519 | # Spelling dictionary name. Available dictionaries: none. To make it work, 520 | # install the 'python-enchant' package. 521 | spelling-dict= 522 | 523 | # List of comma separated words that should be considered directives if they 524 | # appear at the beginning of a comment and should not be checked. 525 | spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: 526 | 527 | # List of comma separated words that should not be checked. 528 | spelling-ignore-words= 529 | 530 | # A path to a file that contains the private dictionary; one word per line. 531 | spelling-private-dict-file= 532 | 533 | # Tells whether to store unknown words to the private dictionary (see the 534 | # --spelling-private-dict-file option) instead of raising a message. 535 | spelling-store-unknown-words=no 536 | 537 | 538 | [STRING] 539 | 540 | # This flag controls whether inconsistent-quotes generates a warning when the 541 | # character used as a quote delimiter is used inconsistently within a module. 542 | check-quote-consistency=no 543 | 544 | # This flag controls whether the implicit-str-concat should generate a warning 545 | # on implicit string concatenation in sequences defined over several lines. 546 | check-str-concat-over-line-jumps=no 547 | 548 | 549 | [TYPECHECK] 550 | 551 | # List of decorators that produce context managers, such as 552 | # contextlib.contextmanager. Add to this list to register other decorators that 553 | # produce valid context managers. 554 | contextmanager-decorators=contextlib.contextmanager 555 | 556 | # List of members which are set dynamically and missed by pylint inference 557 | # system, and so shouldn't trigger E1101 when accessed. Python regular 558 | # expressions are accepted. 559 | generated-members= 560 | 561 | # Tells whether to warn about missing members when the owner of the attribute 562 | # is inferred to be None. 563 | ignore-none=yes 564 | 565 | # This flag controls whether pylint should warn about no-member and similar 566 | # checks whenever an opaque object is returned when inferring. The inference 567 | # can return multiple potential results while evaluating a Python object, but 568 | # some branches might not be evaluated, which results in partial inference. In 569 | # that case, it might be useful to still emit no-member and other checks for 570 | # the rest of the inferred objects. 571 | ignore-on-opaque-inference=yes 572 | 573 | # List of symbolic message names to ignore for Mixin members. 574 | ignored-checks-for-mixins=no-member, 575 | not-async-context-manager, 576 | not-context-manager, 577 | attribute-defined-outside-init 578 | 579 | # List of class names for which member attributes should not be checked (useful 580 | # for classes with dynamically set attributes). This supports the use of 581 | # qualified names. 582 | ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace 583 | 584 | # Show a hint with possible names when a member name was not found. The aspect 585 | # of finding the hint is based on edit distance. 586 | missing-member-hint=yes 587 | 588 | # The minimum edit distance a name should have in order to be considered a 589 | # similar match for a missing member name. 590 | missing-member-hint-distance=1 591 | 592 | # The total number of similar names that should be taken in consideration when 593 | # showing a hint for a missing member. 594 | missing-member-max-choices=1 595 | 596 | # Regex pattern to define which classes are considered mixins. 597 | mixin-class-rgx=.*[Mm]ixin 598 | 599 | # List of decorators that change the signature of a decorated function. 600 | signature-mutators= 601 | 602 | 603 | [VARIABLES] 604 | 605 | # List of additional names supposed to be defined in builtins. Remember that 606 | # you should avoid defining new builtins when possible. 607 | additional-builtins= 608 | 609 | # Tells whether unused global variables should be treated as a violation. 610 | allow-global-unused-variables=yes 611 | 612 | # List of names allowed to shadow builtins 613 | allowed-redefined-builtins= 614 | 615 | # List of strings which can identify a callback function by name. A callback 616 | # name must start or end with one of those strings. 617 | callbacks=cb_, 618 | _cb 619 | 620 | # A regular expression matching the name of dummy variables (i.e. expected to 621 | # not be used). 622 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 623 | 624 | # Argument names that match this expression will be ignored. 625 | ignored-argument-names=_.*|^ignored_|^unused_ 626 | 627 | # Tells whether we should check for unused import in __init__ files. 628 | init-import=no 629 | 630 | # List of qualified module names which can have objects that can redefine 631 | # builtins. 632 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 633 | 634 | -------------------------------------------------------------------------------- /services/github_service.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=E1136, E1135, W0718, C0411 2 | 3 | import json 4 | import time 5 | from calendar import timegm 6 | from datetime import date, datetime, timedelta, timezone 7 | from time import gmtime, sleep 8 | from typing import Any, Callable 9 | import concurrent.futures 10 | 11 | from dateutil.relativedelta import relativedelta 12 | from github import (Github, NamedUser, RateLimitExceededException, 13 | UnknownObjectException, GithubException) 14 | from github.Organization import Organization 15 | from github.Repository import Repository 16 | from gql import Client, gql 17 | from gql.transport.aiohttp import AIOHTTPTransport 18 | from gql.transport.exceptions import TransportServerError 19 | from requests import Session 20 | 21 | from config.logging_config import logging 22 | 23 | logging.getLogger("gql").setLevel(logging.WARNING) 24 | 25 | 26 | def retries_github_rate_limit_exception_at_next_reset_once(func: Callable) -> Callable: 27 | def decorator(*args, **kwargs): 28 | """ 29 | A decorator to retry the method when rate limiting for GitHub resets if the method fails due to Rate Limit related exception. 30 | 31 | WARNING: Since this decorator retries methods, ensure that the method being decorated is idempotent 32 | or contains only one non-idempotent method at the end of a call chain to GitHub. 33 | 34 | Example of idempotent methods are: 35 | - Retrieving data 36 | Example of (potentially) non-idempotent methods are: 37 | - Deleting data 38 | - Updating data 39 | """ 40 | try: 41 | return func(*args, **kwargs) 42 | except (RateLimitExceededException, TransportServerError) as exception: 43 | logging.warning( 44 | f"Caught {type(exception).__name__}, retrying calls when rate limit resets.") 45 | rate_limits = args[0].github_client_core_api.get_rate_limit() 46 | rate_limit_to_use = rate_limits.core if isinstance( 47 | exception, RateLimitExceededException) else rate_limits.graphql 48 | 49 | reset_timestamp = timegm(rate_limit_to_use.reset.timetuple()) 50 | now_timestamp = timegm(gmtime()) 51 | time_until_core_api_rate_limit_resets = ( 52 | reset_timestamp - now_timestamp) if reset_timestamp > now_timestamp else 0 53 | 54 | wait_time_buffer = 5 55 | sleep(time_until_core_api_rate_limit_resets + 56 | wait_time_buffer if time_until_core_api_rate_limit_resets else 0) 57 | return func(*args, **kwargs) 58 | 59 | return decorator 60 | 61 | 62 | class GithubService: 63 | USER_ACCESS_REMOVED_ISSUE_TITLE: str = "User access removed, access is now via a team" 64 | GITHUB_GQL_MAX_PAGE_SIZE = 100 65 | GITHUB_GQL_DEFAULT_PAGE_SIZE = 80 66 | ENTERPRISE_NAME = "ministry-of-justice-uk" 67 | 68 | # Added to stop TypeError on instantiation. See https://github.com/python/cpython/blob/d2340ef25721b6a72d45d4508c672c4be38c67d3/Objects/typeobject.c#L4444 69 | def __new__(cls, *_, **__): 70 | return super(GithubService, cls).__new__(cls) 71 | 72 | def __init__(self, org_token: str, organisation_name: str, 73 | enterprise_name: str = ENTERPRISE_NAME) -> None: 74 | self.organisation_name: str = organisation_name 75 | self.enterprise_name: str = enterprise_name 76 | self.organisations_in_enterprise: list = ["ministryofjustice", "moj-analytical-services"] 77 | 78 | self.github_client_core_api: Github = Github(org_token) 79 | self.github_client_gql_api: Client = Client(transport=AIOHTTPTransport( 80 | url="https://api.github.com/graphql", 81 | headers={"Authorization": f"Bearer {org_token}"}, 82 | ), execute_timeout=120) 83 | self.github_client_rest_api = Session() 84 | self.github_client_rest_api.headers.update( 85 | { 86 | "Accept": "application/vnd.github+json", 87 | "Authorization": f"Bearer {org_token}", 88 | } 89 | ) 90 | 91 | @retries_github_rate_limit_exception_at_next_reset_once 92 | def get_outside_collaborators_login_names(self) -> list[str]: 93 | logging.info("Getting Outside Collaborators Login Names") 94 | outside_collaborators = self.github_client_core_api.get_organization( 95 | self.organisation_name).get_outside_collaborators() or [] 96 | return outside_collaborators 97 | 98 | @retries_github_rate_limit_exception_at_next_reset_once 99 | def add_all_users_to_team(self, team_name: str) -> None: 100 | logging.info(f"Adding all users to {team_name}") 101 | team_id = self.get_team_id_from_team_name(team_name) 102 | all_users = self.__get_all_users() 103 | existing_users_in_team = self.__get_users_from_team(team_id) 104 | 105 | for user in all_users: 106 | if user not in existing_users_in_team: 107 | self.__add_user_to_team(user, team_id) 108 | 109 | @retries_github_rate_limit_exception_at_next_reset_once 110 | def __get_all_users(self) -> list: 111 | logging.info("Getting all organization members") 112 | logging.info(self.github_client_core_api.get_rate_limit()) 113 | return self.github_client_core_api.get_organization(self.organisation_name).get_members() or [] 114 | 115 | @retries_github_rate_limit_exception_at_next_reset_once 116 | def __add_user_to_team(self, user: NamedUser, team_id: int) -> None: 117 | logging.info(f"Adding user {user.login} to team {team_id}") 118 | try: 119 | self.github_client_core_api.get_organization(self.organisation_name).get_team(team_id).add_membership(user) 120 | except GithubException as err: 121 | print(f"Could not add {user.login} to team {team_id}: {err}") 122 | 123 | @retries_github_rate_limit_exception_at_next_reset_once 124 | def __get_repositories_from_team(self, team_id: int) -> list[Repository]: 125 | logging.info(f"Getting all repositories for team {team_id}") 126 | return self.github_client_core_api.get_organization(self.organisation_name).get_team( 127 | team_id).get_repos() or [] 128 | 129 | @retries_github_rate_limit_exception_at_next_reset_once 130 | def __get_users_from_team(self, team_id: int) -> list: 131 | logging.info(f"Getting all named users for team {team_id}") 132 | return self.github_client_core_api.get_organization(self.organisation_name).get_team( 133 | team_id).get_members() or [] 134 | 135 | @retries_github_rate_limit_exception_at_next_reset_once 136 | def get_team_id_from_team_name(self, team_name: str) -> int | TypeError: 137 | logging.info(f"Getting team ID for team name {team_name}") 138 | data = self.github_client_gql_api.execute(gql(""" 139 | query($organisation_name: String!, $team_name: String!) { 140 | organization(login: $organisation_name) { 141 | team(slug: $team_name) { 142 | databaseId 143 | } 144 | } 145 | } 146 | """), variable_values={"organisation_name": self.organisation_name, "team_name": team_name}) 147 | 148 | return data["organization"]["team"]["databaseId"] 149 | 150 | @retries_github_rate_limit_exception_at_next_reset_once 151 | def get_paginated_list_of_org_repository_names(self, after_cursor: str | None, 152 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE) -> dict[str, Any]: 153 | logging.info( 154 | f"Getting paginated list of org repository names. Page size {page_size}, after cursor {bool(after_cursor)}") 155 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 156 | raise ValueError( 157 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 158 | return self.github_client_gql_api.execute(gql(""" 159 | query($organisation_name: String!, $page_size: Int!, $after_cursor: String) { 160 | organization(login: $organisation_name) { 161 | repositories(first: $page_size, after: $after_cursor, isLocked: false, isArchived: false) { 162 | pageInfo { 163 | endCursor 164 | hasNextPage 165 | } 166 | edges { 167 | node { 168 | isDisabled 169 | name 170 | } 171 | } 172 | } 173 | } 174 | } 175 | """), variable_values={"organisation_name": self.organisation_name, "page_size": page_size, 176 | "after_cursor": after_cursor}) 177 | 178 | @retries_github_rate_limit_exception_at_next_reset_once 179 | def get_paginated_list_of_repositories_per_type(self, repo_type: str, after_cursor: str | None, 180 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE) -> dict[str, Any]: 181 | logging.info( 182 | f"Getting paginated list of repositories per type {repo_type}. Page size {page_size}, after cursor {bool(after_cursor)}") 183 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 184 | raise ValueError( 185 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 186 | the_query = f"org:{self.organisation_name}, archived:false, is:{repo_type}" 187 | return self.github_client_gql_api.execute(gql(""" 188 | query($page_size: Int!, $after_cursor: String, $the_query: String!) { 189 | search( 190 | type: REPOSITORY 191 | query: $the_query 192 | first: $page_size 193 | after: $after_cursor 194 | ) { 195 | repos: edges { 196 | repo: node { 197 | ... on Repository { 198 | isDisabled 199 | isPrivate 200 | isLocked 201 | name 202 | pushedAt 203 | url 204 | description 205 | hasIssuesEnabled 206 | repositoryTopics(first: 10) { 207 | edges { 208 | node { 209 | topic { 210 | name 211 | } 212 | } 213 | } 214 | } 215 | defaultBranchRef { 216 | name 217 | } 218 | collaborators(affiliation: DIRECT) { 219 | totalCount 220 | } 221 | licenseInfo { 222 | name 223 | } 224 | collaborators(affiliation: DIRECT) { 225 | totalCount 226 | } 227 | branchProtectionRules(first: 10) { 228 | edges { 229 | node { 230 | isAdminEnforced 231 | pattern 232 | requiredApprovingReviewCount 233 | requiresApprovingReviews 234 | } 235 | } 236 | } 237 | } 238 | } 239 | } 240 | pageInfo { 241 | hasNextPage 242 | endCursor 243 | } 244 | } 245 | } 246 | """), variable_values={"the_query": the_query, "page_size": page_size, "after_cursor": after_cursor}) 247 | 248 | @retries_github_rate_limit_exception_at_next_reset_once 249 | def get_paginated_list_of_team_names(self, after_cursor: str | None, 250 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE) -> dict[str, Any]: 251 | logging.info( 252 | f"Getting paginated list of team names. Page size {page_size}, after cursor {bool(after_cursor)}") 253 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 254 | raise ValueError( 255 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 256 | return self.github_client_gql_api.execute(gql(""" 257 | query($organisation_name: String!, $page_size: Int!, $after_cursor: String) { 258 | organization(login: $organisation_name) { 259 | teams(first: $page_size, after:$after_cursor) { 260 | pageInfo { 261 | endCursor 262 | hasNextPage 263 | } 264 | edges { 265 | node { 266 | slug 267 | } 268 | } 269 | } 270 | } 271 | } 272 | """), variable_values={ 273 | "organisation_name": self.organisation_name, 274 | "page_size": page_size, 275 | "after_cursor": after_cursor 276 | }) 277 | 278 | @retries_github_rate_limit_exception_at_next_reset_once 279 | def get_paginated_list_of_team_repositories(self, team_name: str, after_cursor: str | None, 280 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE) -> dict[str, Any]: 281 | logging.info( 282 | f"Getting paginated list of team repos. Page size {page_size}, after cursor {bool(after_cursor)}") 283 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 284 | raise ValueError( 285 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 286 | return self.github_client_gql_api.execute(gql(""" 287 | query($organisation_name: String!, $team_name: String!, $page_size: Int!, $after_cursor: String) { 288 | organization(login: $organisation_name) { 289 | team(slug: $team_name) { 290 | repositories(first: $page_size, after:$after_cursor) { 291 | edges { 292 | node { 293 | name 294 | } 295 | } 296 | pageInfo { 297 | endCursor 298 | hasNextPage 299 | } 300 | } 301 | } 302 | } 303 | } 304 | """), variable_values={ 305 | "organisation_name": self.organisation_name, 306 | "team_name": team_name, 307 | "page_size": page_size, 308 | "after_cursor": after_cursor 309 | }) 310 | 311 | @retries_github_rate_limit_exception_at_next_reset_once 312 | def get_team_names(self) -> list[str]: 313 | """A wrapper function to run a GraphQL query to get the team names in the organisation 314 | 315 | Returns: 316 | list: A list of the team names 317 | """ 318 | has_next_page = True 319 | after_cursor = None 320 | team_names = [] 321 | 322 | while has_next_page: 323 | data = self.get_paginated_list_of_team_names(after_cursor, 100) 324 | 325 | if data["organization"]["teams"]["edges"] is not None: 326 | for team in data["organization"]["teams"]["edges"]: 327 | team_names.append(team["node"]["slug"]) 328 | 329 | has_next_page = data["organization"]["teams"]["pageInfo"]["hasNextPage"] 330 | after_cursor = data["organization"]["teams"]["pageInfo"]["endCursor"] 331 | return team_names 332 | 333 | @retries_github_rate_limit_exception_at_next_reset_once 334 | def get_team_repository_names(self, team_name: str) -> list[str]: 335 | """A wrapper function to run a GraphQL query to get a team repository names 336 | 337 | Returns: 338 | list: A list of the team repository names 339 | """ 340 | has_next_page = True 341 | after_cursor = None 342 | team_repository_names = [] 343 | 344 | while has_next_page: 345 | data = self.get_paginated_list_of_team_repositories( 346 | team_name, after_cursor, 100) 347 | 348 | if data["organization"]["team"]["repositories"]["edges"] is not None: 349 | for team in data["organization"]["team"]["repositories"]["edges"]: 350 | team_repository_names.append(team["node"]["name"]) 351 | 352 | has_next_page = data["organization"]["team"]["repositories"]["pageInfo"]["hasNextPage"] 353 | after_cursor = data["organization"]["team"]["repositories"]["pageInfo"]["endCursor"] 354 | return team_repository_names 355 | 356 | @retries_github_rate_limit_exception_at_next_reset_once 357 | def get_team_user_names(self, team_name: str) -> list[str]: 358 | """A wrapper function to run a GraphQL query to get a team user names 359 | 360 | Returns: 361 | list: A list of the team user names 362 | """ 363 | has_next_page = True 364 | after_cursor = None 365 | team_user_names = [] 366 | 367 | while has_next_page: 368 | data = self.get_paginated_list_of_team_user_names( 369 | team_name, after_cursor, 100) 370 | 371 | if data["organization"]["team"]["members"]["edges"] is not None: 372 | for team in data["organization"]["team"]["members"]["edges"]: 373 | team_user_names.append(team["node"]["login"]) 374 | 375 | has_next_page = data["organization"]["team"]["members"]["pageInfo"]["hasNextPage"] 376 | after_cursor = data["organization"]["team"]["members"]["pageInfo"]["endCursor"] 377 | return team_user_names 378 | 379 | @retries_github_rate_limit_exception_at_next_reset_once 380 | def get_org_repo_names(self) -> list[str]: 381 | """A wrapper function to run a GraphQL query to get a list of the organisation repository names 382 | (open repositories only). 383 | 384 | Returns: 385 | list: A list of the organisation repository names 386 | """ 387 | has_next_page = True 388 | after_cursor = None 389 | repository_names = [] 390 | while has_next_page: 391 | data = self.get_paginated_list_of_org_repository_names( 392 | after_cursor, 100) 393 | 394 | if data["organization"]["repositories"]["edges"] is not None: 395 | for repo in data["organization"]["repositories"]["edges"]: 396 | if repo["node"]["isDisabled"]: 397 | continue 398 | repository_names.append(repo["node"]["name"]) 399 | 400 | has_next_page = data["organization"]["repositories"]["pageInfo"]["hasNextPage"] 401 | after_cursor = data["organization"]["repositories"]["pageInfo"]["endCursor"] 402 | return repository_names 403 | 404 | @retries_github_rate_limit_exception_at_next_reset_once 405 | def check_circleci_config_in_repos(self) -> list[str]: 406 | """Check if each repository in the list has a CircleCI configuration file using GraphQL. 407 | 408 | Args: 409 | repo_list (list): A list of repository names. 410 | 411 | Returns: 412 | list: A list of repository names that have a CircleCI configuration file. 413 | """ 414 | has_next_page = True 415 | after_cursor = None 416 | repos_with_circleci_config = [] 417 | 418 | while has_next_page: 419 | data = self.get_paginated_circleci_config_check(after_cursor, 100) 420 | 421 | if data["organization"]["repositories"]["edges"]: 422 | for repo in data["organization"]["repositories"]["edges"]: 423 | if repo["node"]["object"]: 424 | repos_with_circleci_config.append(repo["node"]["name"]) 425 | 426 | has_next_page = data["organization"]["repositories"]["pageInfo"]["hasNextPage"] 427 | after_cursor = data["organization"]["repositories"]["pageInfo"]["endCursor"] 428 | return repos_with_circleci_config 429 | 430 | def get_paginated_circleci_config_check(self, after_cursor: str | None, page_size: int) -> dict[str, Any]: 431 | logging.info(f"Checking CircleCI config in repos. Page size {page_size}, after cursor {bool(after_cursor)}") 432 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 433 | raise ValueError(f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 434 | 435 | return self.github_client_gql_api.execute(gql(""" 436 | query($organisation_name: String!, $page_size: Int!, $after_cursor: String) { 437 | organization(login: $organisation_name) { 438 | repositories(first: $page_size, after: $after_cursor) { 439 | pageInfo { 440 | endCursor 441 | hasNextPage 442 | } 443 | edges { 444 | node { 445 | name 446 | object(expression: "HEAD:.circleci/config.yml") { 447 | ... on Blob { 448 | id 449 | } 450 | } 451 | } 452 | } 453 | } 454 | } 455 | } 456 | """), variable_values={ 457 | "organisation_name": self.organisation_name, 458 | "page_size": page_size, 459 | "after_cursor": after_cursor 460 | }) 461 | 462 | @retries_github_rate_limit_exception_at_next_reset_once 463 | def get_paginated_list_of_unlocked_unarchived_repos_and_their_first_100_outside_collaborators( 464 | self, 465 | after_cursor: str | None, 466 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE, 467 | ) -> dict[str, Any]: 468 | logging.info( 469 | f"Getting paginated list of org unlocked unarchived repositories and their first 100 outside collaborators. Page size {page_size}, after cursor {bool(after_cursor)}" 470 | ) 471 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 472 | raise ValueError( 473 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 474 | return self.github_client_gql_api.execute(gql(""" 475 | query($organisation_name: String!, $page_size: Int!, $after_cursor: String) { 476 | organization(login: $organisation_name) { 477 | repositories(first: $page_size, after: $after_cursor, isLocked: false, isArchived: false) { 478 | pageInfo { 479 | endCursor 480 | hasNextPage 481 | } 482 | nodes { 483 | name 484 | isDisabled 485 | visibility 486 | collaborators(first: 100, affiliation: OUTSIDE){ 487 | pageInfo { 488 | hasNextPage 489 | } 490 | edges { 491 | node { 492 | login 493 | } 494 | } 495 | } 496 | } 497 | 498 | } 499 | } 500 | } 501 | """), variable_values={"organisation_name": self.organisation_name, "page_size": page_size, 502 | "after_cursor": after_cursor}) 503 | 504 | @retries_github_rate_limit_exception_at_next_reset_once 505 | def get_active_repos_and_outside_collaborators(self) -> list[dict[str, bool, list[str]]]: 506 | """A wrapper function to run a GraphQL query to get a list of dictionaries containing active 507 | repositories and for each its set of current affiliated Outside Collaborators login names for 508 | the organisaiton. These Outside Collaborators are affiliated with at least one active (not locked, 509 | archived nor disabled) repository. 510 | 511 | Output: 512 | list: A list of dictionaries containing active repositories and for each its list of current 513 | affiliated outside collaborators login names. Also includes whether the repo has public 514 | visibility or not (False = private or internal) 515 | [ 516 | {'repository': 'repo1', 'public': True, 'outside_collaborators': ['c1', 'c2', 'c3']}, 517 | {'repository': 'repo2', 'public': False, 'outside_collaborators': ['c3', 'c4']} 518 | ] 519 | """ 520 | repo_has_next_page = True 521 | after_cursor = None 522 | active_repos_and_outside_collaborators = [] 523 | while repo_has_next_page: 524 | data = self.get_paginated_list_of_unlocked_unarchived_repos_and_their_first_100_outside_collaborators( 525 | after_cursor, self.GITHUB_GQL_MAX_PAGE_SIZE 526 | ) 527 | if data["organization"]["repositories"]["nodes"] is not None: 528 | for repo in data["organization"]["repositories"]["nodes"]: 529 | if repo["isDisabled"]: 530 | continue 531 | # The query only returns the first 100 Outside Collaborators on a repo, if there is a next page 532 | # of collaborators it will not collect them. This is very unlikely to occur, however the function 533 | # output is unreliable if it does so. 534 | if repo["collaborators"]["pageInfo"]["hasNextPage"]: 535 | raise ValueError( 536 | f"Repo has more than {self.GITHUB_GQL_MAX_PAGE_SIZE} Outside Collaborators; only collected the first {self.GITHUB_GQL_MAX_PAGE_SIZE} hence omitted and calculation unreliable." 537 | ) 538 | public = False 539 | if repo["visibility"] == "PUBLIC": 540 | public = True 541 | if repo["collaborators"]["edges"]: 542 | collaborators = [edge["node"]["login"] for edge in repo["collaborators"]["edges"]] 543 | active_repos_and_outside_collaborators.append( 544 | {"repository": repo["name"], "public": public, "outside_collaborators": collaborators} 545 | ) 546 | repo_has_next_page = data["organization"]["repositories"]["pageInfo"]["hasNextPage"] 547 | after_cursor = data["organization"]["repositories"]["pageInfo"]["endCursor"] 548 | 549 | return active_repos_and_outside_collaborators 550 | 551 | @retries_github_rate_limit_exception_at_next_reset_once 552 | def fetch_all_repositories_in_org(self) -> list[dict[str, Any]]: 553 | """A wrapper function to run a GraphQL query to get the list of repositories in the organisation 554 | Returns: 555 | list: A list of the organisation repos names 556 | """ 557 | repos = [] 558 | 559 | # Specifically switch off logging for this query as it is very large and doesn't need to be logged 560 | logging.disabled = True 561 | 562 | for repo_type in ["public", "private", "internal"]: 563 | after_cursor = None 564 | has_next_page = True 565 | while has_next_page: 566 | data = self.get_paginated_list_of_repositories_per_type( 567 | repo_type, after_cursor) 568 | 569 | if data["search"]["repos"] is not None: 570 | for repo in data["search"]["repos"]: 571 | if repo["repo"]["isDisabled"] or repo["repo"]["isLocked"]: 572 | continue 573 | repos.append(repo["repo"]) 574 | 575 | has_next_page = data["search"]["pageInfo"]["hasNextPage"] 576 | after_cursor = data["search"]["pageInfo"]["endCursor"] 577 | 578 | # Re-enable logging 579 | logging.disabled = False 580 | return repos 581 | 582 | @retries_github_rate_limit_exception_at_next_reset_once 583 | def get_paginated_list_of_team_user_names(self, team_name: str, after_cursor: str | None, 584 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE) -> dict[str, Any]: 585 | 586 | logging.info( 587 | f"Getting paginated list of team user names. Page size {page_size}, after cursor {bool(after_cursor)}") 588 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 589 | raise ValueError( 590 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 591 | return self.github_client_gql_api.execute(gql(""" 592 | query($organisation_name: String!, $team_name: String!, $page_size: Int!, $after_cursor: String) { 593 | organization(login: $organisation_name) { 594 | team(slug: $team_name) { 595 | members(first: $page_size, after: $after_cursor) { 596 | edges { 597 | node { 598 | login 599 | } 600 | } 601 | pageInfo { 602 | hasNextPage 603 | endCursor 604 | } 605 | } 606 | } 607 | } 608 | } 609 | """), variable_values={ 610 | "organisation_name": self.organisation_name, 611 | "team_name": team_name, 612 | "page_size": page_size, 613 | "after_cursor": after_cursor 614 | }) 615 | 616 | @retries_github_rate_limit_exception_at_next_reset_once 617 | def get_repository_direct_users(self, repository_name: str) -> list: 618 | users = self.github_client_core_api.get_repo( 619 | f"{self.organisation_name}/{repository_name}").get_collaborators("direct") or [] 620 | return [member.login.lower() for member in users] 621 | 622 | @retries_github_rate_limit_exception_at_next_reset_once 623 | def get_repository_collaborators(self, repository_name: str) -> list: 624 | users = self.github_client_core_api.get_repo( 625 | f"{self.organisation_name}/{repository_name}").get_collaborators("outside") or [] 626 | return [member.login.lower() for member in users] 627 | 628 | @retries_github_rate_limit_exception_at_next_reset_once 629 | def get_paginated_list_of_repositories_per_topic(self, topic: str, after_cursor: str | None, 630 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE) -> dict[str, Any]: 631 | """ 632 | Fetches a paginated list of repositories associated with a given GitHub topic/ 633 | 634 | Parameters: 635 | - topic (str): The GitHub topic for which to fetch associated repositories. 636 | - after_cursor (str | None): The pagination cursor to fetch results after a certain point. If None, fetches from the beginning. 637 | - page_size (int, optional): The number of repository results to return per page. Defaults to GITHUB_GQL_DEFAULT_PAGE_SIZE. 638 | Note that there's an upper limit, GITHUB_GQL_MAX_PAGE_SIZE, beyond which an exception will be raised. 639 | 640 | Returns: 641 | - dict[str, Any]: A dictionary containing the repository data and pagination information. 642 | 643 | Raises: 644 | - ValueError: If the specified page size exceeds GitHub's maximum allowable page size. 645 | 646 | Usage: 647 | >>> get_paginated_list_of_repositories_per_topic('standards-compliant', None, 50) 648 | { ...repository data... } 649 | 650 | """ 651 | logging.info( 652 | f"Getting paginated list of repositories per topic {topic}. Page size {page_size}, after cursor {bool(after_cursor)}") 653 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 654 | raise ValueError( 655 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 656 | the_query = f"org:{self.organisation_name}, archived:false, topic:{topic}" 657 | query = gql(""" 658 | query($page_size: Int!, $after_cursor: String, $the_query: String!) { 659 | search( 660 | type: REPOSITORY 661 | query: $the_query 662 | first: $page_size 663 | after: $after_cursor 664 | ) { 665 | repos: edges { 666 | repo: node { 667 | ... on Repository { 668 | name 669 | isDisabled 670 | isLocked 671 | hasIssuesEnabled 672 | repositoryTopics(first: 10) { 673 | edges { 674 | node { 675 | topic { 676 | name 677 | } 678 | } 679 | } 680 | } 681 | collaborators(affiliation: DIRECT) { 682 | totalCount 683 | } 684 | } 685 | } 686 | } 687 | pageInfo { 688 | hasNextPage 689 | endCursor 690 | } 691 | } 692 | } 693 | """) 694 | variable_values = {"the_query": the_query, "page_size": page_size, 695 | "after_cursor": after_cursor} 696 | return self.github_client_gql_api.execute(query, variable_values) 697 | 698 | @retries_github_rate_limit_exception_at_next_reset_once 699 | def get_user_org_email_address(self, user_name) -> str | None: 700 | data = self.github_client_gql_api.execute(gql(""" 701 | query($organisation_name: String!, $user_name: String!) { 702 | user(login: $user_name) { 703 | organizationVerifiedDomainEmails(login: $organisation_name) 704 | } 705 | } 706 | """), variable_values={"organisation_name": self.organisation_name, "user_name": user_name}) 707 | 708 | if data["user"]["organizationVerifiedDomainEmails"]: 709 | return data["user"]["organizationVerifiedDomainEmails"][0] 710 | return None 711 | 712 | @retries_github_rate_limit_exception_at_next_reset_once 713 | def get_org_members_login_names(self) -> list[str]: 714 | logging.info("Getting Org Members Login Names") 715 | members = self.github_client_core_api.get_organization( 716 | self.organisation_name).get_members() or [] 717 | return [member.login.lower() for member in members] 718 | 719 | def enterprise_audit_activity_for_user(self, username: str): 720 | response_okay = 200 721 | url = f"https://api.github.com/enterprises/{self.enterprise_name}/audit-log?phrase=actor%3A{username}" 722 | response = self.github_client_rest_api.get(url, timeout=10) 723 | if response.status_code == response_okay: 724 | return json.loads(response.content.decode("utf-8")) 725 | raise ValueError( 726 | f"Failed to get audit activity for user {username}. Response status code: {response.status_code}") 727 | 728 | @retries_github_rate_limit_exception_at_next_reset_once 729 | def _get_user_from_audit_log(self, username: str): 730 | logging.info("Getting User from Audit Log") 731 | response_okay = 200 732 | url = f"https://api.github.com/orgs/{self.organisation_name}/audit-log?phrase=actor%3A{username}" 733 | response = self.github_client_rest_api.get(url, timeout=10) 734 | if response.status_code == response_okay: 735 | return json.loads(response.content.decode("utf-8")) 736 | return 0 737 | 738 | @retries_github_rate_limit_exception_at_next_reset_once 739 | def get_audit_log_active_users(self, users: list) -> list: 740 | three_months_ago_date = datetime.now() - relativedelta(months=3) 741 | 742 | active_users = [] 743 | for user in users: 744 | audit_log_data = self._get_user_from_audit_log( 745 | user["username"]) 746 | # No data means the user has no activity in the audit log 747 | if len(audit_log_data) > 0: 748 | last_active_date = datetime.fromtimestamp( 749 | audit_log_data[0]["@timestamp"] / 1000.0) 750 | if last_active_date > three_months_ago_date: 751 | active_users.append(user["username"].lower()) 752 | return active_users 753 | 754 | def get_last_audit_log_activity_date_for_user(self, username: str) -> datetime | None: 755 | audit_activity = self.enterprise_audit_activity_for_user(username) 756 | if audit_activity: 757 | return datetime.fromtimestamp(audit_activity[0]["@timestamp"] / 1000.0) 758 | return None 759 | 760 | @retries_github_rate_limit_exception_at_next_reset_once 761 | def check_dormant_users_audit_activity_since_date(self, users: list, since_date: datetime) -> list: 762 | return [user for user in users if self.is_user_dormant_since_date(user, since_date)] 763 | 764 | def is_user_dormant_since_date(self, user: str, since_date: datetime) -> bool: 765 | audit_activity = self.enterprise_audit_activity_for_user(user) 766 | if audit_activity: 767 | last_active_date = datetime.fromtimestamp( 768 | audit_activity[0]["@timestamp"] / 1000.0) 769 | if last_active_date < since_date: 770 | logging.info( 771 | f"User {user} last active date: {last_active_date}, adding to dormant users list") 772 | return True 773 | else: 774 | logging.info( 775 | f"User {user} has no audit activity, adding to dormant users list") 776 | return True 777 | return False 778 | 779 | @retries_github_rate_limit_exception_at_next_reset_once 780 | def remove_user_from_gitub(self, user: str): 781 | github_user = self.github_client_core_api.get_user(user) 782 | self.github_client_core_api.get_organization( 783 | self.organisation_name).remove_from_membership(github_user) 784 | 785 | def get_inactive_users(self, team_name: str, users_to_ignore, repositories_to_ignore: list[str], 786 | inactivity_months: int) -> list[NamedUser.NamedUser]: 787 | """ 788 | Identifies and returns a list of inactive users from a specified GitHub team based on a given inactivity period. 789 | 790 | :param team_name: The name of the GitHub team to check for inactive users. 791 | :type team_name: str 792 | :param users_to_ignore: A list of usernames to ignore during the inactivity check. 793 | :type users_to_ignore: list[str] 794 | :param repositories_to_ignore: A list of repository names to exclude from the inactivity check. 795 | :type repositories_to_ignore: list[str] 796 | :param inactivity_months: The threshold for user inactivity, specified in months. Users inactive for longer than this period are considered inactive. 797 | :type inactivity_months: int 798 | :return: A list of NamedUser objects representing the users who are identified as inactive. 799 | :rtype: list[NamedUser.NamedUser] 800 | 801 | Example Usage: 802 | inactive_users = get_inactive_users("operations-engineering", ["user1"], ["repo1"], 18) 803 | """ 804 | team_id = self.get_team_id_from_team_name(team_name) 805 | logging.info( 806 | f"Identifying inactive users in team {team_name}, id = {team_id}") 807 | users = self._get_unignored_users_from_team(team_id, users_to_ignore) 808 | repositories = self._get_repositories_managed_by_team( 809 | team_id, repositories_to_ignore) 810 | return self._identify_inactive_users(users, repositories, inactivity_months) 811 | 812 | def _identify_inactive_users(self, users: list[NamedUser.NamedUser], repositories: list[Repository], 813 | inactivity_months: int) -> list[NamedUser.NamedUser]: 814 | users_to_remove = [] 815 | for user in users: 816 | if self._is_user_inactive(user, repositories, inactivity_months): 817 | logging.info( 818 | f"User {user.login} is inactive for {inactivity_months} months") 819 | users_to_remove.append(user) 820 | return users_to_remove 821 | 822 | def _get_unignored_users_from_team(self, team_id: int, users_to_ignore: list[str]) -> list[NamedUser.NamedUser]: 823 | logging.info(f"Ignoring users {users_to_ignore}") 824 | users = self.__get_users_from_team(team_id) 825 | return [user for user in users if user.login not in users_to_ignore] 826 | 827 | def _get_repositories_managed_by_team(self, team_id: int, repositories_to_ignore: list[str]) -> list[Repository]: 828 | logging.info(f"Ignoring repositories {repositories_to_ignore}") 829 | repositories = self.__get_repositories_from_team(team_id) 830 | return [repo for repo in repositories if repo.name.lower() not in repositories_to_ignore] 831 | 832 | def _is_user_inactive(self, user: NamedUser.NamedUser, repositories: list[Repository], 833 | inactivity_months: int) -> bool: 834 | cutoff_date = datetime.now() - timedelta(days=inactivity_months * 835 | 30) # Roughly calculate the cutoff date 836 | 837 | for repo in repositories: 838 | # Get the user's commits in the repo 839 | try: 840 | commits = repo.get_commits(author=user) 841 | except Exception: 842 | logging.error( 843 | f"An exception occurred while getting commits for user {user.login} in repo {repo.name}") 844 | continue 845 | 846 | # Check if any commit is later than the cutoff date 847 | try: 848 | for commit in commits: 849 | if commit.commit.author.date > cutoff_date: 850 | return False # User has been active in this repo, so not considered inactive 851 | except Exception: 852 | logging.error( 853 | f"An exception occurred while getting commit date for user {user.login} in repo {repo.name}") 854 | continue 855 | 856 | return True # User is inactive in all given repositories 857 | 858 | @retries_github_rate_limit_exception_at_next_reset_once 859 | def _get_paginated_organization_members_with_emails(self, after_cursor: str | None, 860 | page_size: int = GITHUB_GQL_MAX_PAGE_SIZE) -> dict[str, Any]: 861 | logging.info( 862 | f"Getting paginated organization members with emails. Page size {page_size}, after cursor {bool(after_cursor)}") 863 | 864 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 865 | raise ValueError( 866 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 867 | 868 | query = gql(""" 869 | query($org: String!, $page_size: Int!, $after_cursor: String) { 870 | organization(login: $org) { 871 | membersWithRole(first: $page_size, after: $after_cursor) { 872 | nodes { 873 | login 874 | organizationVerifiedDomainEmails(login: $org) 875 | } 876 | pageInfo { 877 | hasNextPage 878 | endCursor 879 | } 880 | } 881 | } 882 | } 883 | """) 884 | 885 | variable_values = { 886 | "org": self.organisation_name, 887 | "page_size": page_size, 888 | "after_cursor": after_cursor 889 | } 890 | 891 | return self.github_client_gql_api.execute(query, variable_values) 892 | 893 | def get_github_member_list(self): 894 | github_usernames = [] 895 | next_page = True 896 | after_cursor = None 897 | all_members = [] 898 | 899 | while next_page: 900 | response = self._get_paginated_organization_members_with_emails( 901 | after_cursor=after_cursor) 902 | 903 | if 'errors' in response: 904 | logging.error( 905 | f"Error retrieving organization members: {response['errors']}") 906 | break 907 | 908 | if 'organization' in response and 'membersWithRole' in response['organization']: 909 | all_members = response['organization']['membersWithRole']['nodes'] 910 | member_data = response['organization']['membersWithRole'] 911 | 912 | for member in all_members: 913 | email = member['organizationVerifiedDomainEmails'][0] if member[ 914 | 'organizationVerifiedDomainEmails'] else None 915 | github_usernames.append({ 916 | "username": member["login"], 917 | "email": email 918 | }) 919 | next_page = member_data['pageInfo']['hasNextPage'] 920 | if next_page: 921 | after_cursor = member_data['pageInfo']['endCursor'] 922 | else: 923 | next_page = False 924 | 925 | return github_usernames 926 | 927 | def get_remaining_licences(self) -> int: 928 | """ 929 | Fetches the number of remaining licenses in the GitHub Enterprise. 930 | 931 | Returns: 932 | int: The number of remaining licenses. 933 | """ 934 | licence = self.github_client_core_api.get_enterprise( 935 | self.enterprise_name).get_consumed_licenses() 936 | return licence.total_seats_purchased - licence.total_seats_consumed 937 | 938 | @retries_github_rate_limit_exception_at_next_reset_once 939 | def update_team_repository_permission(self, team_name: str, repositories, permission: str): 940 | org = self.github_client_core_api.get_organization( 941 | self.organisation_name) 942 | 943 | try: 944 | team = org.get_team_by_slug(team_name) 945 | except UnknownObjectException as exc: 946 | raise ValueError( 947 | f"Team '{team_name}' does not exist in organization '{self.organisation_name}'") from exc 948 | 949 | for repo_name in repositories: 950 | try: 951 | repo = org.get_repo(repo_name) 952 | except UnknownObjectException as exc: 953 | raise ValueError( 954 | f"Repository '{repo_name}' does not exist in organization '{self.organisation_name}'") from exc 955 | 956 | logging.info( 957 | f"Updating {team_name} team's permission to {permission} on {repo_name}") 958 | team.set_repo_permission(repo, permission) 959 | 960 | def flag_owner_permission_changes(self, since_date: str) -> list: 961 | recent_changes = self.audit_log_member_changes(since_date) 962 | list_of_changes_to_flag = [] 963 | for change in recent_changes: 964 | match change["action"]: 965 | case "org.add_member" if change["permission"] == "ADMIN": 966 | list_of_changes_to_flag.append(change) 967 | case "org.update_member" if change["permission"] == "ADMIN" and change["permissionWas"] == "READ": 968 | list_of_changes_to_flag.append(change) 969 | case _: 970 | logging.info( 971 | f"Change {change} does not meet criteria to flag") 972 | 973 | return list_of_changes_to_flag 974 | 975 | @retries_github_rate_limit_exception_at_next_reset_once 976 | def audit_log_member_changes(self, since_date: str) -> list: 977 | logging.info(f"Getting audit log entries since {since_date}") 978 | today = datetime.now() 979 | query = """ 980 | query($organisation_name: String!, $since_date: String!, $cursor: String) { 981 | organization(login: $organisation_name) { 982 | auditLog( 983 | first: 100 984 | after: $cursor 985 | query: $since_date 986 | ) { 987 | edges{ 988 | node{ 989 | ... on OrgAddMemberAuditEntry { 990 | action 991 | createdAt 992 | actorLogin 993 | operationType 994 | permission 995 | userLogin 996 | } 997 | ... on OrgUpdateMemberAuditEntry { 998 | action 999 | createdAt 1000 | actorLogin 1001 | operationType 1002 | permission 1003 | permissionWas 1004 | userLogin 1005 | } 1006 | } 1007 | } 1008 | pageInfo { 1009 | endCursor 1010 | hasNextPage 1011 | } 1012 | } 1013 | } 1014 | } 1015 | """ 1016 | variable_values = { 1017 | "organisation_name": self.organisation_name, 1018 | "since_date": f"action:org.add_member action:org.update_member created:{since_date}..{today.strftime('%Y-%m-%d')}", 1019 | "cursor": None 1020 | } 1021 | 1022 | all_entries = [] 1023 | while True: 1024 | data = self.github_client_gql_api.execute( 1025 | gql(query), variable_values=variable_values) 1026 | all_entries.extend( 1027 | [entry["node"] for entry in data["organization"]["auditLog"]["edges"] if entry["node"]]) 1028 | 1029 | if data["organization"]["auditLog"]["pageInfo"]["hasNextPage"]: 1030 | variable_values["cursor"] = data["organization"]["auditLog"]["pageInfo"]["endCursor"] 1031 | else: 1032 | break 1033 | 1034 | return all_entries 1035 | 1036 | @retries_github_rate_limit_exception_at_next_reset_once 1037 | def check_for_audit_log_new_members(self, since_date: str) -> list: 1038 | logging.info( 1039 | f"Getting audit log entries for new members since {since_date}") 1040 | today = datetime.now() 1041 | query = """ 1042 | query($organisation_name: String!, $since_date: String!, $cursor: String) { 1043 | organization(login: $organisation_name) { 1044 | auditLog( 1045 | first: 100 1046 | after: $cursor 1047 | query: $since_date 1048 | ) { 1049 | edges{ 1050 | node{ 1051 | ... on OrgAddMemberAuditEntry { 1052 | action 1053 | createdAt 1054 | actorLogin 1055 | userLogin 1056 | } 1057 | } 1058 | } 1059 | pageInfo { 1060 | endCursor 1061 | hasNextPage 1062 | } 1063 | } 1064 | } 1065 | } 1066 | """ 1067 | variable_values = { 1068 | "organisation_name": self.organisation_name, 1069 | "since_date": f"action:org.add_member created:{since_date}..{today.strftime('%Y-%m-%d')}", 1070 | "cursor": None 1071 | } 1072 | 1073 | new_members = [] 1074 | while True: 1075 | data = self.github_client_gql_api.execute( 1076 | gql(query), variable_values=variable_values) 1077 | new_members.extend( 1078 | [entry["node"] for entry in data["organization"]["auditLog"]["edges"] if entry["node"]]) 1079 | 1080 | if data["organization"]["auditLog"]["pageInfo"]["hasNextPage"]: 1081 | variable_values["cursor"] = data["organization"]["auditLog"]["pageInfo"]["endCursor"] 1082 | else: 1083 | break 1084 | 1085 | return new_members 1086 | 1087 | @retries_github_rate_limit_exception_at_next_reset_once 1088 | def get_all_organisations_in_enterprise(self) -> list[Organization]: 1089 | logging.info( 1090 | f"Getting all organisations for enterprise {self.ENTERPRISE_NAME}") 1091 | 1092 | return [org.login for org in self.github_client_core_api.get_user().get_orgs()] or [] 1093 | 1094 | @retries_github_rate_limit_exception_at_next_reset_once 1095 | def get_gha_minutes_used_for_organisation(self, organization) -> int: 1096 | logging.info( 1097 | f"Getting all github actions minutes used for organization {organization}") 1098 | 1099 | headers = { 1100 | "Accept": "application/vnd.github+json", 1101 | "X-GitHub-Api-Version": "2022-11-28" 1102 | } 1103 | 1104 | response = self.github_client_rest_api.get( 1105 | f"https://api.github.com/orgs/{organization}/settings/billing/actions", headers=headers) 1106 | 1107 | return response.json() 1108 | 1109 | def get_all_private_internal_repos_names(self, org_name: str): 1110 | 1111 | logging.info( 1112 | f"Getting all private and internal repos used for organization {org_name}") 1113 | private_internal_repos = [] 1114 | org = self.github_client_core_api.get_organization(org_name) 1115 | all_repos = org.get_repos(type="all") 1116 | private_internal_repos = [ 1117 | repo.name for repo in all_repos if not repo.archived and repo.visibility in ( 1118 | 'internal', 'private')] 1119 | 1120 | return private_internal_repos 1121 | 1122 | @retries_github_rate_limit_exception_at_next_reset_once 1123 | def get_current_month_gha_minutes_for_enterprise(self, month) -> int: 1124 | 1125 | logging.info( 1126 | f"Getting usage report for current billing month for the enterprise {self.enterprise_name}") 1127 | 1128 | headers = { 1129 | "Accept": "application/vnd.github+json", 1130 | "X-GitHub-Api-Version": "2022-11-28" 1131 | } 1132 | 1133 | max_attempts = 3 1134 | for attempt in range(1, max_attempts + 1): 1135 | response = self.github_client_rest_api.get( 1136 | f"https://api.github.com/enterprises/{self.enterprise_name}/settings/billing/usage?month={month}", 1137 | headers=headers 1138 | ) 1139 | 1140 | data = response.json() 1141 | if data and data.get("usageItems"): 1142 | return data 1143 | 1144 | logging.warning(f"Attempt {attempt} failed to get valid usage data. Retrying...") 1145 | time.sleep(2 ** attempt) # Exponential backoff: 2s, 4s, 8s 1146 | 1147 | raise ValueError("Failed to retrieve valid usage data after multiple attempts.") 1148 | 1149 | @retries_github_rate_limit_exception_at_next_reset_once 1150 | def modify_gha_minutes_quota_threshold(self, new_threshold): 1151 | logging.info(f"Changing the alerting threshold to {new_threshold}%") 1152 | 1153 | headers = { 1154 | "Accept": "application/vnd.github+json", 1155 | "X-GitHub-Api-Version": "2022-11-28" 1156 | } 1157 | 1158 | payload = {'value': str(new_threshold)} 1159 | 1160 | self.github_client_rest_api.patch( 1161 | "https://api.github.com/repos/ministryofjustice/operations-engineering/actions/variables/GHA_MINUTES_QUOTA_THRESHOLD", json.dumps(payload), headers=headers) 1162 | 1163 | @retries_github_rate_limit_exception_at_next_reset_once 1164 | def _get_repository_variable(self, variable_name): 1165 | actions_variable = self.github_client_core_api.get_repo( 1166 | f'{self.organisation_name}/operations-engineering').get_variable(variable_name) 1167 | return actions_variable.value 1168 | 1169 | @retries_github_rate_limit_exception_at_next_reset_once 1170 | def reset_alerting_threshold_if_first_day_of_month(self): 1171 | base_alerting_threshold = int(self._get_repository_variable("GHA_MINUTES_QUOTA_BASE_THRESHOLD")) 1172 | 1173 | if date.today().day == 1: 1174 | self.modify_gha_minutes_quota_threshold(base_alerting_threshold) 1175 | 1176 | @retries_github_rate_limit_exception_at_next_reset_once 1177 | def calculate_total_minutes_enterprise(self): 1178 | 1179 | organisations = self.get_all_organisations_in_enterprise() 1180 | enterprise_billable_repos = [] 1181 | for org in organisations: 1182 | org_repos = self.get_all_private_internal_repos_names(org) 1183 | enterprise_billable_repos.extend(org_repos) 1184 | 1185 | gha_minutes_total = 0.0 1186 | if enterprise_billable_repos: 1187 | current_billing_month = datetime.now().month 1188 | try: 1189 | billing_data = self.get_current_month_gha_minutes_for_enterprise(current_billing_month) 1190 | usage_items = billing_data.get('usageItems', []) 1191 | 1192 | if not usage_items: 1193 | logging.warning("No usage items returned in billing data.") 1194 | else: 1195 | gha_minutes_total = sum( 1196 | item['quantity'] 1197 | for item in usage_items 1198 | if item.get('product') == 'actions' 1199 | and item.get('unitType') == 'Minutes' 1200 | and item.get('repositoryName') in enterprise_billable_repos 1201 | ) 1202 | except Exception as e: 1203 | logging.error(f"Failed to retrieve or process billing data: {e}") 1204 | 1205 | return gha_minutes_total 1206 | 1207 | @retries_github_rate_limit_exception_at_next_reset_once 1208 | def check_if_gha_minutes_quota_is_low(self): 1209 | 1210 | total_minutes_used = self.calculate_total_minutes_enterprise() 1211 | 1212 | print(f"Total minutes used: {total_minutes_used}") 1213 | 1214 | total_quota = int(self._get_repository_variable("GHA_MINUTES_QUOTA_TOTAL")) 1215 | 1216 | percentage_used = (total_minutes_used / total_quota) * 100 1217 | 1218 | self.reset_alerting_threshold_if_first_day_of_month() 1219 | 1220 | threshold = int(self._get_repository_variable("GHA_MINUTES_QUOTA_THRESHOLD")) 1221 | 1222 | if percentage_used >= threshold: 1223 | return {'threshold': threshold, 'percentage_used': percentage_used} 1224 | return False 1225 | 1226 | @retries_github_rate_limit_exception_at_next_reset_once 1227 | def get_new_pat_creation_events_for_organization(self) -> list: 1228 | logging.info( 1229 | f"Fetching PATs for the {self.organisation_name} organisation...") 1230 | 1231 | headers = { 1232 | "Accept": "application/vnd.github+json", 1233 | "X-GitHub-Api-Version": "2022-11-28" 1234 | } 1235 | 1236 | url = f"https://api.github.com/orgs/{self.organisation_name}/personal-access-tokens" 1237 | 1238 | response = self.github_client_rest_api.get(url, headers=headers) 1239 | 1240 | if response.status_code == 200: 1241 | logging.info("Successfully retrieved PAT list.") 1242 | return response.json() 1243 | 1244 | raise ValueError(f"Failed to fetch PAT list: {response.status_code}, error: {response}") 1245 | 1246 | @retries_github_rate_limit_exception_at_next_reset_once 1247 | def calculate_repo_age(self, repo: str) -> list: 1248 | creation_date = self.github_client_core_api.get_repo(f"{self.organisation_name}/{repo}").created_at 1249 | 1250 | age_in_days = (datetime.now(timezone.utc) - creation_date).days 1251 | 1252 | return age_in_days 1253 | 1254 | @retries_github_rate_limit_exception_at_next_reset_once 1255 | def get_old_poc_repositories(self) -> list: 1256 | poc_repositories = [repo['repo']['name'] for repo in self.get_paginated_list_of_repositories_per_topic("poc", None)['search']['repos']] 1257 | 1258 | old_poc_repositories = {} 1259 | age_threshold = 30 1260 | 1261 | for repo in poc_repositories: 1262 | age = self.calculate_repo_age(repo) 1263 | if age >= age_threshold: 1264 | old_poc_repositories[repo] = age 1265 | 1266 | return old_poc_repositories 1267 | 1268 | @retries_github_rate_limit_exception_at_next_reset_once 1269 | def get_user_removal_events(self, since_date: str, actor: str) -> list: 1270 | logging.info(f"Getting audit log entries for users removed by {actor} since {since_date}") 1271 | today = datetime.now() 1272 | query_string = f"action:org.remove_member actor:{actor} created:{since_date}..{today.strftime('%Y-%m-%d')}" 1273 | 1274 | query = """ 1275 | query($organisation_name: String!, $query_string: String!, $cursor: String) { 1276 | organization(login: $organisation_name) { 1277 | auditLog( 1278 | first: 100 1279 | after: $cursor 1280 | query: $query_string 1281 | ) { 1282 | edges{ 1283 | node{ 1284 | ... on OrgRemoveMemberAuditEntry { 1285 | action 1286 | createdAt 1287 | actorLogin 1288 | userLogin 1289 | } 1290 | } 1291 | } 1292 | pageInfo { 1293 | endCursor 1294 | hasNextPage 1295 | } 1296 | } 1297 | } 1298 | } 1299 | """ 1300 | variable_values = { 1301 | "organisation_name": self.organisation_name, 1302 | "query_string": query_string, 1303 | "cursor": None 1304 | } 1305 | 1306 | removed_users = [] 1307 | while True: 1308 | data = self.github_client_gql_api.execute( 1309 | gql(query), variable_values=variable_values) 1310 | removed_users.extend( 1311 | [entry["node"] for entry in data["organization"]["auditLog"]["edges"] if entry["node"]]) 1312 | if data["organization"]["auditLog"]["pageInfo"]["hasNextPage"]: 1313 | variable_values["cursor"] = data["organization"]["auditLog"]["pageInfo"]["endCursor"] 1314 | else: 1315 | break 1316 | 1317 | return removed_users 1318 | 1319 | @retries_github_rate_limit_exception_at_next_reset_once 1320 | def get_current_contributors_for_repo( 1321 | self, 1322 | repo_name: str, 1323 | current_logins: list) -> dict[str, set[str]]: 1324 | """ 1325 | Input: 1326 | repo_name: string 1327 | current_logins: list of logins for the GitHub org 1328 | 1329 | Output: 1330 | Returns a dictionary containing the active repo name and its set of 1331 | contributors who are also in the org current users set. 1332 | {'repository': 'repo1', 'contributors': {'c1', 'c2', 'c3'}} 1333 | Returns None if repo has 0 contributors or 0 current contributors. 1334 | """ 1335 | repo = self.github_client_core_api.get_repo(f"{self.organisation_name}/{repo_name}") 1336 | contributors = [contributor.login for contributor in repo.get_contributors()] 1337 | if contributors: 1338 | current_contributors = set(current_logins).intersection(set(contributors)) 1339 | if current_contributors: 1340 | return {"repository": repo_name, "contributors": current_contributors} 1341 | 1342 | return None 1343 | 1344 | @retries_github_rate_limit_exception_at_next_reset_once 1345 | def get_current_contributors_for_active_repos(self) -> list[dict[str, set[str]]]: 1346 | """ 1347 | Returns a sorted list of dictionaries containing the active repo name and 1348 | its set of contributors who are also in the org current users set. Output 1349 | is sorted by number of current contributors in descending order. 1350 | [ 1351 | {'repository': 'repo1', 'contributors': {'c1', 'c2', 'c3'}}, 1352 | {'repository': 'repo2', 'contributors': {'c3', 'c4'}} 1353 | ] 1354 | Repos with 0 contributors or 0 current contributors are dropped. 1355 | """ 1356 | logins = [user.login for user in self.__get_all_users()] 1357 | active_repos = self.get_active_repositories() 1358 | number_of_repos = len(active_repos) 1359 | logging.info( 1360 | f"Org: {self.organisation_name} has {len(logins)} members and {number_of_repos} active repositories" 1361 | ) 1362 | 1363 | active_repos_and_current_contributors = [] 1364 | 1365 | logging.info(f"Getting current contributors for active repos in {self.organisation_name}") 1366 | logging.info(f"Pre getting current contributors: {self.github_client_core_api.get_rate_limit()}") 1367 | with concurrent.futures.ThreadPoolExecutor() as executor: 1368 | futures = [] 1369 | for repo_name in active_repos: 1370 | futures.append( 1371 | executor.submit( 1372 | self.get_current_contributors_for_repo, 1373 | repo_name=repo_name, 1374 | current_logins=logins 1375 | ) 1376 | ) 1377 | for future in concurrent.futures.as_completed(futures): 1378 | if future.result(): 1379 | active_repos_and_current_contributors.append(future.result()) 1380 | 1381 | sorted_active_repos_and_current_contributors = sorted( 1382 | active_repos_and_current_contributors, 1383 | key=lambda d: len(d['contributors']), 1384 | reverse=True 1385 | ) 1386 | logging.info(f"Post getting current contributors: {self.github_client_core_api.get_rate_limit()}") 1387 | return sorted_active_repos_and_current_contributors 1388 | 1389 | @retries_github_rate_limit_exception_at_next_reset_once 1390 | def get_repos_user_has_contributed_to( 1391 | self, 1392 | username: str, 1393 | repos_and_contributors: list[dict[str, set[str]]] 1394 | ) -> list[str]: 1395 | """ 1396 | For a known GH user get the repos they have contributed to within org. 1397 | """ 1398 | repos = [ 1399 | repo_object.get("repository") for repo_object in repos_and_contributors if username in repo_object.get("contributors") 1400 | ] 1401 | return repos 1402 | 1403 | @retries_github_rate_limit_exception_at_next_reset_once 1404 | def user_has_committed_to_repo_since( 1405 | self, 1406 | username: str, 1407 | repo_name: str, 1408 | since_datetime: datetime 1409 | ) -> bool: 1410 | 1411 | repo = self.github_client_core_api.get_repo(f"{self.organisation_name}/{repo_name}") 1412 | commits = repo.get_commits( 1413 | since=since_datetime, 1414 | author=username.lower() 1415 | ) 1416 | if commits.totalCount > 0: 1417 | return True 1418 | return False 1419 | 1420 | @retries_github_rate_limit_exception_at_next_reset_once 1421 | def user_has_commmits_since( 1422 | self, 1423 | username: str, 1424 | repos_and_contributors: list[dict[str, set[str]]], 1425 | since_datetime: datetime 1426 | ) -> bool: 1427 | """ 1428 | Determine if a given user has made any commits to at least one of the repos they 1429 | have contributed to in the org since the given datetime. 1430 | """ 1431 | repos = self.get_repos_user_has_contributed_to( 1432 | username=username, 1433 | repos_and_contributors=repos_and_contributors 1434 | ) 1435 | 1436 | for repo_name in repos: 1437 | repo = self.github_client_core_api.get_repo(f"{self.organisation_name}/{repo_name}") 1438 | commits = repo.get_commits( 1439 | since=since_datetime, 1440 | author=username.lower() 1441 | ) 1442 | if commits.totalCount > 0: 1443 | return True 1444 | 1445 | return False 1446 | 1447 | @retries_github_rate_limit_exception_at_next_reset_once 1448 | def get_paginated_list_of_unlocked_unarchived_repos( 1449 | self, 1450 | after_cursor: str | None, 1451 | page_size: int = GITHUB_GQL_DEFAULT_PAGE_SIZE, 1452 | ) -> dict[str, Any]: 1453 | logging.info( 1454 | f"Getting paginated list of org unlocked unarchived repositories. Page size {page_size}, after cursor {bool(after_cursor)}" 1455 | ) 1456 | if page_size > self.GITHUB_GQL_MAX_PAGE_SIZE: 1457 | raise ValueError( 1458 | f"Page size of {page_size} is too large. Max page size {self.GITHUB_GQL_MAX_PAGE_SIZE}") 1459 | return self.github_client_gql_api.execute(gql(""" 1460 | query($organisation_name: String!, $page_size: Int!, $after_cursor: String) { 1461 | organization(login: $organisation_name) { 1462 | repositories(first: $page_size, after: $after_cursor, isLocked: false, isArchived: false) { 1463 | pageInfo { 1464 | endCursor 1465 | hasNextPage 1466 | } 1467 | nodes { 1468 | name 1469 | isDisabled 1470 | } 1471 | 1472 | } 1473 | } 1474 | } 1475 | """), variable_values={"organisation_name": self.organisation_name, "page_size": page_size, 1476 | "after_cursor": after_cursor}) 1477 | 1478 | @retries_github_rate_limit_exception_at_next_reset_once 1479 | def get_active_repositories(self) -> list[str]: 1480 | """A wrapper function to run a GraphQL query to get a list of active (not locked, 1481 | not archived nor disabled) repositories in the organisation. 1482 | 1483 | Returns: 1484 | list: A list of the organisation's active repositories. 1485 | """ 1486 | 1487 | repo_has_next_page = True 1488 | after_cursor = None 1489 | active_repositories = [] 1490 | while repo_has_next_page: 1491 | data = self.get_paginated_list_of_unlocked_unarchived_repos( 1492 | after_cursor, self.GITHUB_GQL_MAX_PAGE_SIZE 1493 | ) 1494 | if data["organization"]["repositories"]["nodes"] is not None: 1495 | for repo in data["organization"]["repositories"]["nodes"]: 1496 | if repo["isDisabled"]: 1497 | continue 1498 | active_repositories.append(repo["name"]) 1499 | repo_has_next_page = data["organization"]["repositories"]["pageInfo"]["hasNextPage"] 1500 | after_cursor = data["organization"]["repositories"]["pageInfo"]["endCursor"] 1501 | 1502 | return active_repositories 1503 | --------------------------------------------------------------------------------