├── src ├── audit_rules │ ├── passwd.rules │ ├── shadow.rules │ └── sudoers.rules ├── constants.py ├── auditd_templates │ └── auditd.conf.j2 ├── utils.py ├── loki_alert_rules │ └── audit.yaml ├── workloads.py └── charm.py ├── .gitignore ├── renovate.json ├── .github ├── CODEOWNERS └── workflows │ ├── release_charm_artifacts.yaml │ └── check.yaml ├── justfile ├── README.md ├── CONTRIBUTING.md ├── charmcraft.yaml ├── tests └── unit │ ├── test_utils.py │ ├── test_workloads.py │ └── test_charm.py ├── pyproject.toml ├── lib └── charms │ ├── operator_libs_linux │ ├── v1 │ │ └── systemd.py │ └── v0 │ │ └── apt.py │ └── grafana_agent │ └── v0 │ └── cos_agent.py └── LICENSE /src/audit_rules/passwd.rules: -------------------------------------------------------------------------------- 1 | -w /etc/passwd -p wa -k passwd_changes 2 | -------------------------------------------------------------------------------- /src/audit_rules/shadow.rules: -------------------------------------------------------------------------------- 1 | -w /etc/shadow -p wa -k shadow_changes 2 | -------------------------------------------------------------------------------- /src/audit_rules/sudoers.rules: -------------------------------------------------------------------------------- 1 | -w /etc/sudoers -p wa -k sudoers_changes 2 | -w /etc/sudoers.d/ -p wa -k sudoers_d_changes 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | build/ 3 | *.charm 4 | .tox/ 5 | .coverage 6 | __pycache__/ 7 | *.py[cod] 8 | .idea 9 | .vscode/ 10 | report 11 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /src/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Canonical Ltd. 2 | # See LICENSE file for licensing details. 3 | 4 | """Common of globals variables to the charm.""" 5 | 6 | # Auditd rules 7 | AUDIT_RULE_PATH = "./src/audit_rules" 8 | 9 | # Template files 10 | TEMPLATE_FILE_PATH = "./src/auditd_templates" 11 | AUDITD_CONFIG_TEMPLATE = "auditd.conf.j2" 12 | 13 | # Common constants 14 | AUDITD_MIN_NUM_LOGS = 0 15 | AUDITD_MAX_NUM_LOGS = 999 16 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This file is centrally managed as a template file in https://github.com/canonical/solutions-engineering-automation 2 | # To update the file: 3 | # - Edit it in the canonical/solutions-engineering-automation repository. 4 | # - Open a PR with the changes. 5 | # - When the PR merges, the soleng-terraform bot will open a PR to the target repositories with the changes. 6 | # 7 | # These owners will be the default owners for everything in the repo. Unless a 8 | # later match takes precedence, @canonical/soleng-reviewers will be requested for 9 | # review when someone opens a pull request. 10 | * @canonical/soleng-reviewers 11 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | [private] 2 | default: 3 | @just --list 4 | 5 | # Run static and unit-tests recipes 6 | check: static unit-tests 7 | 8 | # Format the Python code 9 | fix: 10 | uv run codespell -w . 11 | uv run ruff format . 12 | uv run ruff check --fix --exit-zero --silent . 13 | 14 | # Run static code analysis 15 | static: 16 | uv run codespell . 17 | uv run ruff format --diff . 18 | uv run ruff check --no-fix . 19 | uv run mypy --install-types --non-interactive . 20 | 21 | # Run unit tests 22 | unit-tests: 23 | uv run pytest ./tests/unit/ \ 24 | -v \ 25 | --cov \ 26 | --cov-report=term-missing \ 27 | --cov-report=html \ 28 | --cov-report=xml 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | A Juju charm that deploys and manages [`auditd`][1] on machine. [`auditd`][1] is the userspace 4 | component to the Linux Auditing System. It's responsible for writing audit records to the disk. 5 | 6 | ## Platform Requirements 7 | 8 | This charm can only be deployed on **bare metal machines or virtual machines**. It **cannot** be 9 | deployed on Linux containers (LXC). 10 | 11 | [`auditd`][1] performs kernel-level auditing and requires direct access to the kernel's audit 12 | subsystem, which is not available within containers. The charm will automatically prevent 13 | deployment on unsupported platforms (LXC containers) and raise an error during installation. 14 | 15 | [1]: https://manpages.ubuntu.com/manpages/noble/man8/auditd.8.html 16 | -------------------------------------------------------------------------------- /src/auditd_templates/auditd.conf.j2: -------------------------------------------------------------------------------- 1 | # 2 | # This file controls the configuration of the audit daemon 3 | # 4 | # Note: This file is managed by Juju, modification to this file will not be persisted. 5 | # 6 | 7 | action_mail_acct = root 8 | admin_space_left = 50 9 | admin_space_left_action = SUSPEND 10 | disk_error_action = SUSPEND 11 | disk_full_action = SUSPEND 12 | distribute_network = no 13 | end_of_event_timeout = 2 14 | flush = INCREMENTAL_ASYNC 15 | freq = 50 16 | krb5_principal = auditd 17 | local_events = yes 18 | log_file = /var/log/audit/audit.log 19 | log_format = ENRICHED 20 | log_group = root 21 | max_log_file = {{ max_log_file }} 22 | max_log_file_action = ROTATE 23 | max_restarts = 10 24 | name_format = NONE 25 | num_logs = {{ num_logs }} 26 | overflow_action = SYSLOG 27 | plugin_dir = /etc/audit/plugins.d 28 | priority_boost = 4 29 | q_depth = 2000 30 | space_left = 75 31 | space_left_action = SYSLOG 32 | tcp_client_max_idle = 0 33 | tcp_listen_queue = 5 34 | tcp_max_per_addr = 1 35 | transport = TCP 36 | use_libwrap = yes 37 | verify_email = yes 38 | write_logs = yes 39 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | To contribute to this charm, you will have to prepare the development environment by installing the 4 | following tools. 5 | 6 | - [uv](https://docs.astral.sh/uv/) 7 | - [just](https://github.com/casey/just) 8 | - [Juju](https://documentation.ubuntu.com/juju/3.6/howto/manage-your-juju-deployment/set-up-your-juju-deployment/) 9 | - [Charmcraft](https://documentation.ubuntu.com/charmcraft/stable/howto/) 10 | 11 | ## Testing 12 | 13 | This project uses `uv` for managing test environments. There are some pre-configured recipes that 14 | can be used for linting and formatting code when you're preparing contributions to the charm. You 15 | can learn more about the recipes by running `just --help` or `just`: 16 | 17 | ```shell 18 | $ just 19 | Available recipes: 20 | check # Run fix, static, and unittest recipes 21 | fix # Format the Python code 22 | static # Run static code analysis 23 | unittest # Run unit tests 24 | ``` 25 | 26 | ## Build the charm 27 | 28 | Build the charm in this git repository using: 29 | 30 | ```shell 31 | charmcraft pack 32 | ``` 33 | -------------------------------------------------------------------------------- /charmcraft.yaml: -------------------------------------------------------------------------------- 1 | name: auditd 2 | type: charm 3 | base: ubuntu@24.04 4 | summary: A Juju charm that deploys and manages Linux audit daemon. 5 | description: | 6 | A Juju charm that deploys and manages auditd on the machine. auditd is the userspace component 7 | to the Linux Auditing System. It's responsible for writing audit records to the disk. 8 | 9 | platforms: 10 | amd64: 11 | 12 | subordinate: true 13 | 14 | parts: 15 | charm: 16 | source: . 17 | plugin: uv 18 | build-snaps: [astral-uv] 19 | uv-groups: 20 | - charmlibs 21 | 22 | config: 23 | options: 24 | num_logs: 25 | type: int 26 | default: 10 27 | description: | 28 | The number of auditd log files to retains. If the number is < 2, logs are not rotated. This 29 | number must be 999 or less. 30 | max_log_file: 31 | type: int 32 | default: 512 33 | description: | 34 | The maximum file size in megabytes. When this limit is reached, it will trigger a 35 | 'ROTATE' action to rotate the log file. 36 | 37 | provides: 38 | cos-agent: 39 | interface: cos_agent 40 | limit: 1 41 | 42 | requires: 43 | general-info: 44 | interface: juju-info 45 | scope: container 46 | -------------------------------------------------------------------------------- /tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | from subprocess import CalledProcessError 2 | from unittest.mock import MagicMock, patch 3 | 4 | import pytest 5 | 6 | import utils 7 | 8 | 9 | def test_read_file(tmp_path): 10 | file = tmp_path / "test.txt" 11 | file.write_text("hello", encoding="utf-8") 12 | assert utils.read_file(file) == "hello" 13 | 14 | 15 | @patch("utils.os.chmod") 16 | @patch("utils.pwd.getpwnam") 17 | @patch("utils.os.chown") 18 | def test_write_file(mock_chown, mock_getpwnam, mock_chmod, tmp_path): 19 | file = tmp_path / "test.txt" 20 | mock_getpwnam.return_value = MagicMock(pw_uid=1000, pw_gid=1000) 21 | utils.write_file(file, "data", "root", 0o600) 22 | assert file.read_text(encoding="utf-8") == "data" 23 | mock_chmod.assert_called_once() 24 | mock_getpwnam.assert_called_once_with("root") 25 | mock_chown.assert_called_once() 26 | 27 | 28 | @patch("utils.Environment") 29 | def test_render_jinja2_template(mock_env): 30 | mock_template = MagicMock() 31 | mock_template.render.return_value = "rendered" 32 | mock_env.return_value.get_template.return_value = mock_template 33 | result = utils.render_jinja2_template({"foo": "bar"}, "template", "/path") 34 | assert result == "rendered" 35 | mock_env.return_value.get_template.assert_called_once_with("template") 36 | mock_template.render.assert_called_once_with({"foo": "bar"}) 37 | 38 | 39 | @patch("utils.subprocess.check_output", return_value=b"qemu") 40 | def test_get_machine_virt_type_success(mock_check_output): 41 | assert utils.get_machine_virt_type() == "qemu" 42 | mock_check_output.assert_called_once_with(["systemd-detect-virt"]) 43 | 44 | 45 | @patch("utils.subprocess.check_output", side_effect=CalledProcessError(1, "systemd-detect-virt")) 46 | def test_get_machine_virt_type_failure(_): 47 | with pytest.raises(CalledProcessError): 48 | utils.get_machine_virt_type() 49 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Canonical Ltd. 2 | # See LICENSE file for licensing details. 3 | 4 | """The charm utilities module.""" 5 | 6 | import logging 7 | import os 8 | import pwd 9 | import subprocess 10 | from pathlib import Path 11 | 12 | from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def read_file(path: Path) -> str: 18 | """Read the content of a file. 19 | 20 | Args: 21 | path: Path object to the file. 22 | 23 | Returns: 24 | str: The content of the file. 25 | 26 | """ 27 | return path.read_text(encoding="utf-8") 28 | 29 | 30 | def write_file(path: Path, content: str, owner: str, mode: int = 0o600) -> None: 31 | """Write a content rendered from a template to a file. 32 | 33 | Args: 34 | path: Path object to the file. 35 | content: the data to be written to the file. 36 | owner: the owner of the file. 37 | mode: access permission mask applied to the file using chmod (default=0o600) 38 | 39 | """ 40 | path.write_text(content, encoding="utf-8") 41 | os.chmod(path, mode) 42 | u = pwd.getpwnam(owner) 43 | os.chown(path, uid=u.pw_uid, gid=u.pw_gid) 44 | 45 | 46 | def render_jinja2_template(context: dict, template_name: str, template_file_path: str) -> str: 47 | """Render the jinja2 template file with context. 48 | 49 | Args: 50 | context: dictionary of context pass to the template. 51 | template_name: the name of the template. 52 | template_file_path: the path to find the template files. 53 | 54 | Returns: 55 | str: the rendered content. 56 | 57 | """ 58 | env = Environment( 59 | loader=FileSystemLoader(template_file_path), 60 | autoescape=select_autoescape(), 61 | undefined=StrictUndefined, 62 | keep_trailing_newline=True, 63 | trim_blocks=True, 64 | lstrip_blocks=True, 65 | ) 66 | template = env.get_template(template_name) 67 | rendered_content = template.render(context) 68 | return rendered_content 69 | 70 | 71 | def get_machine_virt_type() -> str: 72 | """Get the machine_virt_type.""" 73 | try: 74 | virt_type = subprocess.check_output(["systemd-detect-virt"]).decode().strip() 75 | except subprocess.CalledProcessError as e: 76 | logger.error("Failed to detect virtualization type: %s", e.stderr) 77 | raise e 78 | return virt_type 79 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "auditd" 3 | version = "0.1.0" 4 | requires-python = ">=3.10" 5 | dependencies = [ 6 | "cosl>=0.0.50", 7 | "jinja2>=3.1.6", 8 | "pydantic>=2.11.7", 9 | ] 10 | 11 | [dependency-groups] 12 | dev = [ 13 | "codespell>=2.4.1", 14 | "mypy>=1.17.1", 15 | "ops[testing]>=3.1.0", 16 | "pytest>=8.4.1", 17 | "pytest-cov>=6.2.1", 18 | "ruff>=0.12.10", 19 | ] 20 | charmlibs = [ 21 | "opentelemetry-api>=1.36.0", 22 | "ops>=3.1.0", 23 | ] 24 | 25 | # 26 | # Tools configuration 27 | # 28 | 29 | [tool.ruff] 30 | line-length = 99 31 | indent-width = 4 32 | exclude = [ 33 | ".eggs", 34 | ".git", 35 | ".mypy_cache", 36 | ".ruff_cache", 37 | ".tox", 38 | ".venv", 39 | "__pypackages__", 40 | "_build", 41 | "build", 42 | "dist", 43 | "venv", 44 | "report", 45 | "lib", 46 | ] 47 | 48 | [tool.ruff.format] 49 | indent-style = "space" 50 | line-ending = "auto" 51 | quote-style = "double" 52 | docstring-code-format = false 53 | skip-magic-trailing-comma = false 54 | docstring-code-line-length = "dynamic" 55 | 56 | 57 | [tool.ruff.lint] 58 | select = [ 59 | "ARG", # flake8-unused-arguments 60 | "B", # flake8-bugbear 61 | "C", # flake8-comprehensions 62 | "C9", # mccabe 63 | "D", # pydocstyle 64 | "E", # pycodestyle errors 65 | "F", # pyflakes 66 | "I", # isort 67 | "N", # pep8-naming 68 | "PL", # pylint 69 | "W", # pycodestyle warnings 70 | ] 71 | ignore = [ 72 | "D203", # one-blank-line-before-class 73 | "D213", # multi-line-summary-second-line 74 | ] 75 | fixable = ["ALL"] 76 | unfixable = [] 77 | per-file-ignores = {"tests/**" = ["ARG","D100","D101","D102","D103","D104"]} 78 | 79 | [tool.ruff.lint.pylint] 80 | max-args = 10 81 | 82 | [tool.codespell] 83 | skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.coverage,report" 84 | ignore-words-list = "assertIn" 85 | 86 | [tool.mypy] 87 | warn_unused_ignores = true 88 | warn_unused_configs = true 89 | warn_unreachable = true 90 | disallow_untyped_defs = true 91 | ignore_missing_imports = true 92 | exclude = [".eggs", ".git", ".tox", ".venv", ".build", "build", "report", "tests", "lib"] 93 | 94 | [tool.pytest.ini_options] 95 | pythonpath = ["./src", "./lib"] 96 | 97 | [tool.coverage.run] 98 | relative_files = true 99 | source = ["."] 100 | omit = ["tests/**", "docs/**", "lib/**", "build/**"] 101 | 102 | [tool.coverage.report] 103 | fail_under = 100 104 | show_missing = true 105 | 106 | [tool.coverage.html] 107 | directory = "tests/unit/report/html" 108 | 109 | [tool.coverage.xml] 110 | output = "tests/unit/report/coverage.xml" 111 | -------------------------------------------------------------------------------- /.github/workflows/release_charm_artifacts.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Release Charm 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | build: 10 | name: Build Charm 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 120 13 | outputs: 14 | charm-name: ${{ steps.rename.outputs.charm }} 15 | steps: 16 | - uses: actions/checkout@v6 17 | with: 18 | submodules: true 19 | 20 | - uses: canonical/craft-actions/charmcraft/setup@main 21 | with: 22 | channel: "3.x/stable" 23 | 24 | - name: Pack the charm 25 | run: | 26 | charmcraft -v pack 27 | 28 | - name: Append the charm name with version tag 29 | id: rename 30 | run: | 31 | ORIGINAL_CHARM=$(ls -1 *.charm) 32 | BASE_NAME="${ORIGINAL_CHARM%.charm}" 33 | VERSIONED_CHARM="${BASE_NAME}-${{ github.ref_name }}.charm" 34 | mv "$ORIGINAL_CHARM" "$VERSIONED_CHARM" 35 | echo "charm=$VERSIONED_CHARM" >> "$GITHUB_OUTPUT" 36 | 37 | - name: Output the name of the built charm 38 | run: echo "::notice::Successfully built ${{ steps.rename.outputs.charm }}" 39 | 40 | - name: Upload the built charm 41 | uses: actions/upload-artifact@v6 42 | with: 43 | name: ${{ steps.rename.outputs.charm }} 44 | path: ${{ steps.rename.outputs.charm }} 45 | 46 | release-and-upload: 47 | name: Create GitHub Release and Upload Charm 48 | needs: build 49 | runs-on: ubuntu-latest 50 | permissions: 51 | contents: write 52 | 53 | steps: 54 | - name: Checkout code 55 | uses: actions/checkout@v6 56 | 57 | - name: Download built charm 58 | uses: actions/download-artifact@v7 59 | with: 60 | name: ${{ needs.build.outputs.charm-name }} 61 | path: ./artifacts 62 | 63 | - name: Create Release and Upload Charm 64 | uses: softprops/action-gh-release@v2 65 | with: 66 | name: Release ${{ github.ref_name }} 67 | generate_release_notes: true 68 | body: | 69 | ## Charm Deployment ${{ github.ref_name }} 70 | 71 | **Download url:** https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}/${{ needs.build.outputs.charm-name }} 72 | **Example:** 73 | ```bash 74 | curl -L https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}/${{ needs.build.outputs.charm-name }} -o auditd.charm 75 | 76 | # Deploy the charm 77 | juju deploy ./auditd.charm 78 | ``` 79 | files: artifacts/${{ needs.build.outputs.charm-name }} 80 | token: ${{ secrets.GITHUB_TOKEN }} 81 | -------------------------------------------------------------------------------- /src/loki_alert_rules/audit.yaml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: linux_audit.rules 3 | rules: 4 | - alert: LinuxUserAuthFailed 5 | expr: | 6 | count by (instance, juju_model, juju_model_uuid) ( 7 | rate({filename=~"/var/log/audit/audit\\.log.*"} 8 | | logfmt 9 | | type = "USER_AUTH" 10 | | res =~ ".*failed.*" [1h]) 11 | ) / ( 12 | count by (instance, juju_model, juju_model_uuid) ( 13 | rate({filename=~"/var/log/audit/audit\\.log.*"} 14 | | logfmt 15 | | type = "USER_AUTH" 16 | | res =~ "(.*failed.*|.*success.*)" [1h]) 17 | ) 18 | ) * 100 > 20 19 | for: 5m 20 | labels: 21 | severity: critical 22 | annotations: 23 | summary: Too many user authentication failed in the past 1 hour 24 | description: | 25 | The user authentication failure rate on {{ $labels.instance }} is > 20% in the past 1 hour, and the situation lasted for 5 minutes. 26 | 27 | LABELS = {{ $labels }} 28 | - alert: LinuxUserLoginFailed 29 | expr: | 30 | count by (instance, juju_model, juju_model_uuid) ( 31 | rate({filename=~"/var/log/audit/audit\\.log.*"} 32 | | logfmt 33 | | type = "USER_LOGIN" 34 | | res =~ ".*failed.*" [1h]) 35 | ) / ( 36 | count by (instance, juju_model, juju_model_uuid) ( 37 | rate({filename=~"/var/log/audit/audit\\.log.*"} 38 | | logfmt 39 | | type = "USER_LOGIN" 40 | | res =~ "(.*failed.*|.*success.*)" [1h]) 41 | ) 42 | ) * 100 > 20 43 | for: 5m 44 | labels: 45 | severity: critical 46 | annotations: 47 | summary: Too many user login failed in the past 1 hour 48 | description: | 49 | The user login failure rate on {{ $labels.instance }} is > 20% in the past 1 hour, and the situation lasted for 5 minutes. 50 | 51 | LABELS = {{ $labels }} 52 | - alert: LinuxUserAccountChanged 53 | expr: | 54 | count by (instance, name, nametype, juju_model, juju_model_uuid) ( 55 | rate({filename=~"/var/log/audit/audit\\.log.*"} 56 | | logfmt 57 | | type = "PATH" 58 | | name =~ "(^/etc/passwd$|^/etc/shadow$|^/etc/sudoers$|^/etc/sudoers.d/$)" 59 | | nametype =~ "(CREATE|DELETE)" [3d]) 60 | ) > 0 61 | for: 5m 62 | labels: 63 | severity: warning 64 | annotations: 65 | summary: Linux user account had been changed in the past 3 days 66 | description: | 67 | The linux user account had been changed on {{ $labels.instance }} in the past 3 days. The file '{{ $labels.name }}' was {{ $labels.nametype }}D. Operators are suggested to check the relevant logs to find out what was changed. 68 | 69 | LABELS = {{ $labels }} 70 | -------------------------------------------------------------------------------- /.github/workflows/check.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | workflow_call: 5 | workflow_dispatch: 6 | pull_request: 7 | types: [opened, synchronize, reopened] 8 | branches: [main] 9 | paths-ignore: 10 | - "**.md" 11 | - "**.rst" 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | code-quality: 19 | name: Code Quality 20 | runs-on: ubuntu-latest 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | python-version: ["3.12"] 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v6 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v6 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | 34 | - name: Install uv 35 | uses: astral-sh/setup-uv@v7 36 | 37 | - name: Install dependencies 38 | run: sudo apt install -y just 39 | 40 | - name: Run static code analysis 41 | run: just static 42 | 43 | loki-alert-rules: 44 | name: Loki Alert Rules 45 | runs-on: ubuntu-latest 46 | strategy: 47 | fail-fast: false 48 | steps: 49 | - name: Checkout 50 | uses: actions/checkout@v6 51 | 52 | - name: Download lokitool 53 | env: 54 | LOKITOOL_VERSION: 3.5.3 55 | run: | 56 | url="https://github.com/grafana/loki/releases/download/v$LOKITOOL_VERSION/lokitool-linux-amd64.zip" 57 | wget "$url" -O ./lokitool.zip 58 | unzip ./lokitool.zip -d lokitool 59 | 60 | - name: Check alert rules 61 | run: | 62 | ./lokitool/lokitool-linux-amd64 rules check --rule-dirs ./src/loki_alert_rules 63 | 64 | unit-tests: 65 | name: Unit Tests 66 | runs-on: ubuntu-latest 67 | strategy: 68 | fail-fast: false 69 | matrix: 70 | python-version: ["3.12"] 71 | steps: 72 | - name: Checkout 73 | uses: actions/checkout@v6 74 | 75 | - name: Set up Python ${{ matrix.python-version }} 76 | uses: actions/setup-python@v6 77 | with: 78 | python-version: ${{ matrix.python-version }} 79 | 80 | - name: Install uv 81 | uses: astral-sh/setup-uv@v7 82 | 83 | - name: Install dependencies 84 | run: sudo apt install -y just 85 | 86 | - name: Run unit tests 87 | run: just unit-tests 88 | 89 | - name: Determine system architecture 90 | run: echo "SYSTEM_ARCH=$(uname -m)" >> $GITHUB_ENV 91 | 92 | - name: Create artifact name suffix 93 | run: | 94 | PYTHON_VERSION_SANITIZED=${{ matrix.python-version }} 95 | PYTHON_VERSION_SANITIZED=${PYTHON_VERSION_SANITIZED//./-} 96 | echo "ARTIFACT_SUFFIX=$PYTHON_VERSION_SANITIZED-${{ env.SYSTEM_ARCH }}" >> $GITHUB_ENV 97 | 98 | - name: Rename Unit Test Coverage Artifact 99 | run: | 100 | if [ -e ".coverage-unit" ]; then 101 | mv .coverage-unit .coverage-unit-${{ env.ARTIFACT_SUFFIX }} 102 | else 103 | echo "No coverage file found, skipping rename" 104 | fi 105 | 106 | - name: Upload Unit Test Coverage File 107 | uses: actions/upload-artifact@v6 108 | with: 109 | include-hidden-files: true 110 | if-no-files-found: ignore 111 | name: coverage-unit-${{ env.ARTIFACT_SUFFIX }} 112 | path: .coverage-unit-${{ env.ARTIFACT_SUFFIX }} 113 | -------------------------------------------------------------------------------- /tests/unit/test_workloads.py: -------------------------------------------------------------------------------- 1 | from subprocess import CalledProcessError 2 | from unittest.mock import MagicMock, patch 3 | 4 | import pytest 5 | from charms.operator_libs_linux.v0 import apt 6 | from charms.operator_libs_linux.v1 import systemd 7 | 8 | from workloads import ( 9 | AuditdConfig, 10 | AuditdService, 11 | AuditdServiceRestartError, 12 | ) 13 | 14 | 15 | def test_auditd_config_valid_num_logs(): 16 | num_logs = 10 17 | max_log_file = 512 18 | config = AuditdConfig(num_logs=num_logs, max_log_file=max_log_file) 19 | assert config.num_logs == num_logs 20 | assert config.max_log_file == max_log_file 21 | 22 | 23 | def test_auditd_config_invalid_num_logs_low(): 24 | with pytest.raises(ValueError): 25 | AuditdConfig(num_logs=-1, max_log_file=512) 26 | 27 | 28 | def test_auditd_config_invalid_num_logs_high(): 29 | with pytest.raises(ValueError): 30 | AuditdConfig(num_logs=1000, max_log_file=512) 31 | 32 | 33 | @patch("workloads.apt.add_package") 34 | @patch("workloads.AuditdService._add_audit_rules") 35 | @patch("workloads.AuditdService._merge_audit_rules") 36 | def test_install_calls_add_package_and_rules(mock_merge, mock_add_rules, mock_add_package): 37 | service = AuditdService() 38 | service.install() 39 | mock_add_package.assert_called_once() 40 | mock_add_rules.assert_called_once() 41 | mock_merge.assert_called_once() 42 | 43 | 44 | @patch("workloads.apt.remove_package") 45 | def test_remove_calls_remove_package(mock_remove_package): 46 | service = AuditdService() 47 | service.remove() 48 | mock_remove_package.assert_called_once() 49 | 50 | 51 | @patch("workloads.systemd.service_restart") 52 | def test_restart_success(mock_restart): 53 | service = AuditdService() 54 | service.restart() 55 | mock_restart.assert_called_once_with(service.name) 56 | 57 | 58 | @patch("workloads.systemd.service_restart", side_effect=systemd.SystemdError) 59 | def test_restart_failure(_): 60 | service = AuditdService() 61 | with pytest.raises(AuditdServiceRestartError): 62 | service.restart() 63 | 64 | 65 | @patch("workloads.write_file") 66 | @patch("workloads.AuditdService.restart") 67 | def test_configure_writes_file_and_restarts(mock_restart, mock_write_file): 68 | service = AuditdService() 69 | service.configure("content") 70 | mock_write_file.assert_called_once() 71 | mock_restart.assert_called_once() 72 | 73 | 74 | @patch("workloads.render_jinja2_template", return_value="rendered") 75 | def test_render_config_returns_rendered(mock_render): 76 | service = AuditdService() 77 | result = service.render_config({"foo": "bar"}) 78 | assert result == "rendered" 79 | mock_render.assert_called_once() 80 | 81 | 82 | @patch("workloads.apt.DebianPackage.from_installed_package") 83 | def test_is_installed_true(_): 84 | service = AuditdService() 85 | assert service.is_installed() is True 86 | 87 | 88 | @patch("workloads.apt.DebianPackage.from_installed_package", side_effect=apt.PackageNotFoundError) 89 | def test_is_installed_false(_): 90 | service = AuditdService() 91 | assert service.is_installed() is False 92 | 93 | 94 | @patch("workloads.systemd.service_running", return_value=True) 95 | def test_is_active_true(_): 96 | service = AuditdService() 97 | assert service.is_active() is True 98 | 99 | 100 | @patch("workloads.systemd.service_running", return_value=False) 101 | def test_is_active_false(_): 102 | service = AuditdService() 103 | assert service.is_active() is False 104 | 105 | 106 | @patch("workloads.read_file", return_value="rule-content") 107 | @patch("workloads.write_file") 108 | @patch("workloads.Path.glob", return_value=[MagicMock(name="rule1", spec=["name"])]) 109 | def test_add_audit_rules(mock_glob, mock_write_file, mock_read_file): 110 | service = AuditdService() 111 | # Patch rule_file.name for the MagicMock 112 | rule_file = mock_glob.return_value[0] 113 | rule_file.name = "rule1" 114 | service._add_audit_rules("/some/path") 115 | mock_read_file.assert_called_once() 116 | mock_write_file.assert_called_once() 117 | 118 | 119 | @patch("workloads.subprocess.run") 120 | def test_merge_audit_rules_success(mock_run): 121 | service = AuditdService() 122 | service._merge_audit_rules() 123 | mock_run.assert_called_once_with(["augenrules", "--load"], check=False) 124 | 125 | 126 | @patch("workloads.subprocess.run", side_effect=CalledProcessError(1, "augenrules --load")) 127 | def test_merge_audit_rules_failure(_): 128 | service = AuditdService() 129 | with pytest.raises(CalledProcessError): 130 | service._merge_audit_rules() 131 | -------------------------------------------------------------------------------- /src/workloads.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Canonical Ltd. 2 | # See LICENSE file for licensing details. 3 | """The auditd service module.""" 4 | 5 | import logging 6 | import subprocess 7 | from pathlib import Path 8 | 9 | import pydantic 10 | from charms.operator_libs_linux.v0 import apt 11 | from charms.operator_libs_linux.v1 import systemd 12 | 13 | from constants import ( 14 | AUDIT_RULE_PATH, 15 | AUDITD_CONFIG_TEMPLATE, 16 | AUDITD_MAX_NUM_LOGS, 17 | AUDITD_MIN_NUM_LOGS, 18 | TEMPLATE_FILE_PATH, 19 | ) 20 | from utils import read_file, render_jinja2_template, write_file 21 | 22 | logger = logging.getLogger() 23 | 24 | 25 | class AuditRuleReloadError(Exception): 26 | """Error when reloading the audit rules.""" 27 | 28 | 29 | class AuditdServiceRestartError(Exception): 30 | """Error when restarting the auditd service.""" 31 | 32 | 33 | class AuditdServiceNotActiveError(Exception): 34 | """Error when the auditd service is not active.""" 35 | 36 | 37 | class AuditdConfig(pydantic.BaseModel): 38 | """Auditd charm configuration.""" 39 | 40 | num_logs: int = pydantic.Field(10) 41 | max_log_file: int = pydantic.Field(512) 42 | 43 | @pydantic.field_validator("num_logs") 44 | @classmethod 45 | def validate_num_logs(cls, value: int) -> int: 46 | """Validate 'num_logs' charm config option.""" 47 | if value < AUDITD_MIN_NUM_LOGS: 48 | raise ValueError(f"'num_logs' cannot be less than {AUDITD_MIN_NUM_LOGS}.") 49 | if value > AUDITD_MAX_NUM_LOGS: 50 | raise ValueError(f"'num_logs' cannot be larger than {AUDITD_MAX_NUM_LOGS}.") 51 | return value 52 | 53 | 54 | class AuditdService: 55 | """Auditd service class.""" 56 | 57 | pkg = "auditd" 58 | name = "auditd" 59 | rule_path = Path("/etc/audit/rules.d/") 60 | config_file = Path("/etc/audit/auditd.conf") 61 | 62 | def install(self) -> None: 63 | """Install the auditd package.""" 64 | apt.add_package(package_names=self.pkg, update_cache=True) 65 | self._add_audit_rules(AUDIT_RULE_PATH) 66 | self._merge_audit_rules() 67 | 68 | def remove(self) -> None: 69 | """Remove the auditd package.""" 70 | apt.remove_package(package_names=self.pkg) 71 | 72 | def restart(self) -> None: 73 | """Restart the auditd service. 74 | 75 | Raises: 76 | AuditdServiceRestartError: When the auditd service fails to restart. 77 | 78 | """ 79 | try: 80 | systemd.service_restart(self.name) 81 | except systemd.SystemdError as exc: 82 | raise AuditdServiceRestartError(f"Failed to restart {self.name}.") from exc 83 | 84 | def configure(self, content: str) -> None: 85 | """Configure auditd service. 86 | 87 | Args: 88 | content (str): The content of auditd configuration. 89 | 90 | Raises: 91 | AuditdServiceRestartError: When the auditd service fails to restart. 92 | 93 | """ 94 | write_file(self.config_file, content, "root", 0o640) 95 | self.restart() 96 | 97 | def render_config(self, context: dict) -> str: 98 | """Render auditd config file given the context. 99 | 100 | Args: 101 | context (dict): The context pass to the template file. 102 | 103 | """ 104 | return render_jinja2_template(context, AUDITD_CONFIG_TEMPLATE, TEMPLATE_FILE_PATH) 105 | 106 | def is_installed(self) -> bool: 107 | """Indicate if auditd is installed. 108 | 109 | Returns: 110 | True if the auditd is installed. 111 | 112 | """ 113 | try: 114 | apt.DebianPackage.from_installed_package(self.pkg) 115 | except apt.PackageNotFoundError: 116 | return False 117 | return True 118 | 119 | def is_active(self) -> bool: 120 | """Indicate if the auditd service is active. 121 | 122 | Returns: 123 | True if the auditd is running. 124 | 125 | """ 126 | return systemd.service_running(self.name) 127 | 128 | def _add_audit_rules(self, path: str) -> None: 129 | """Add audit rule files. 130 | 131 | Args: 132 | path (str): The path to find the rule files. 133 | 134 | """ 135 | for rule_file in Path(path).glob("*"): 136 | content = read_file(rule_file) 137 | destination = self.rule_path / rule_file.name 138 | logger.info("Writing audit rule to '%s'", destination) 139 | write_file(self.rule_path / rule_file.name, content, "root", 0o640) 140 | 141 | def _merge_audit_rules(self) -> None: 142 | """Merge all audit rule files.""" 143 | try: 144 | logger.info("Installing audit rules.") 145 | subprocess.run(["augenrules", "--load"], check=False) 146 | except subprocess.CalledProcessError as e: 147 | logger.error("Failed to reload audit rules: %s", e.stderr) 148 | raise e 149 | -------------------------------------------------------------------------------- /src/charm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2025 Canoincal Ltd. 3 | # See LICENSE file for licensing details. 4 | 5 | """The entrypoint for auditd operator.""" 6 | 7 | import logging 8 | import typing 9 | 10 | import ops 11 | import pydantic 12 | from charms.grafana_agent.v0.cos_agent import COSAgentProvider 13 | 14 | from utils import get_machine_virt_type, read_file 15 | from workloads import AuditdConfig, AuditdService, AuditdServiceRestartError 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class PlatformUnsupportedError(Exception): 21 | """Error when the platform is not supported.""" 22 | 23 | 24 | class AuditdOperatorCharm(ops.CharmBase): 25 | """Auditd service.""" 26 | 27 | def __init__(self, *args: typing.Any) -> None: 28 | """Initialize the instance. 29 | 30 | Args: 31 | args: passthrough to CharmBase. 32 | 33 | """ 34 | super().__init__(*args) 35 | 36 | self.auditd = AuditdService() 37 | 38 | # Forward auditd logs 39 | self.cos_agent_provider = COSAgentProvider( 40 | self, 41 | refresh_events=[self.on.install, self.on.upgrade_charm], 42 | ) 43 | 44 | self.framework.observe(self.on.remove, self._on_remove) 45 | self.framework.observe(self.on.install, self._on_install_or_upgrade) 46 | self.framework.observe(self.on.update_status, self._configure_charm) 47 | self.framework.observe(self.on.upgrade_charm, self._configure_charm) 48 | self.framework.observe(self.on.config_changed, self._configure_charm) 49 | 50 | def _on_remove(self, _: ops.RemoveEvent) -> None: 51 | """Handle remove charm event.""" 52 | if not self._is_valid_platform(): 53 | logger.warning("Not removing package: auditd cannot be run on a linux container.") 54 | return 55 | 56 | self.unit.status = ops.MaintenanceStatus("Removing auditd package.") 57 | self.auditd.remove() 58 | 59 | def _on_install_or_upgrade(self, _: tuple[ops.InstallEvent | ops.UpgradeCharmEvent]) -> None: 60 | """Handle install or upgrade charm event.""" 61 | if not self._is_valid_platform(): 62 | logger.error("Not installing package: auditd cannot be run on a linux container.") 63 | raise PlatformUnsupportedError("Auditd cannot be run on a linux container.") 64 | 65 | self.unit.status = ops.MaintenanceStatus("Installing or upgrading auditd package.") 66 | self.auditd.install() 67 | 68 | def _configure_charm(self, _: ops.HookEvent) -> None: 69 | """Configure the charm idempotently.""" 70 | if not (config := self._get_validated_config()): 71 | self.unit.status = ops.BlockedStatus("Invalid config. Please check `juju debug-log`.") 72 | return 73 | 74 | if not self._configure_auditd(config): 75 | self.unit.status = ops.BlockedStatus("Failed to configure and restart auditd.") 76 | return 77 | 78 | self.unit.status = ops.ActiveStatus() 79 | 80 | def _is_valid_platform(self) -> bool: 81 | """Check if the charm is supported in the current platform. 82 | 83 | Returns: 84 | True if the charm support the current platform, otherwise False. 85 | 86 | """ 87 | if get_machine_virt_type() == "lxc": 88 | logger.error("auditd cannot be run on a linux container.") 89 | return False 90 | return True 91 | 92 | def _get_validated_config(self) -> dict: 93 | """Get validated charm configs. 94 | 95 | Returns: 96 | The validated config (dict), or an empty dict if not validated. 97 | 98 | """ 99 | try: 100 | config = self.load_config(AuditdConfig) 101 | except pydantic.ValidationError as e: 102 | logger.error("Failed to configure auditd service: %s", str(e)) 103 | return {} 104 | return config.model_dump() 105 | 106 | def _configure_auditd(self, config: dict) -> bool: 107 | """Configure auditd. 108 | 109 | Args: 110 | config (dict): The validated charm config. 111 | 112 | Returns: 113 | True if the auditd service is properly configured, otherwise False. 114 | 115 | """ 116 | new_content = self.auditd.render_config(config).strip() 117 | current_content = read_file(AuditdService.config_file).strip() 118 | 119 | if new_content != current_content: 120 | logging.info("Configuring auditd service.") 121 | try: 122 | self.auditd.configure(new_content) 123 | except AuditdServiceRestartError as e: 124 | logger.error("Failed to apply new config: %s", str(e)) 125 | return False 126 | 127 | if not self.auditd.is_active(): 128 | logger.error("Auditd is not active.") 129 | try: 130 | logger.info("Trying to restart auditd.") 131 | self.auditd.restart() 132 | except AuditdServiceRestartError as e: 133 | logger.error("Failed to restart auditd: %s", str(e)) 134 | return False 135 | else: 136 | logger.info("Auditd restart successfully.") 137 | 138 | return True 139 | 140 | 141 | if __name__ == "__main__": # pragma: nocover 142 | ops.main(AuditdOperatorCharm) 143 | -------------------------------------------------------------------------------- /tests/unit/test_charm.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import pytest 4 | from ops import testing 5 | 6 | import charm 7 | from charm import AuditdOperatorCharm 8 | 9 | 10 | @patch("charm.AuditdService.remove") 11 | @patch("charm.get_machine_virt_type", return_value="lxc") 12 | def test_on_remove_lxc(mock_virt, mock_auditd_remove): 13 | ctx = testing.Context(AuditdOperatorCharm) 14 | state = testing.State() 15 | ctx.run(ctx.on.remove(), state) 16 | mock_auditd_remove.assert_not_called() 17 | 18 | 19 | @patch("charm.AuditdService.remove") 20 | @patch("charm.get_machine_virt_type", return_value="kvm") 21 | def test_on_remove_non_lxc(mock_virt, mock_auditd_remove): 22 | ctx = testing.Context(AuditdOperatorCharm) 23 | state = testing.State() 24 | ctx.run(ctx.on.remove(), state) 25 | mock_auditd_remove.assert_called_once() 26 | 27 | 28 | @patch("charm.AuditdService.install") 29 | @patch("charm.get_machine_virt_type", return_value="lxc") 30 | def test_on_install_lxc(mock_virt, mock_auditd_install): 31 | ctx = testing.Context(AuditdOperatorCharm) 32 | state = testing.State() 33 | with pytest.raises(testing.errors.UncaughtCharmError) as e: 34 | ctx.run(ctx.on.install(), state) 35 | mock_auditd_install.assert_not_called() 36 | assert isinstance(e.value.__cause__, charm.PlatformUnsupportedError) 37 | 38 | 39 | @patch("charm.AuditdService.install") 40 | @patch("charm.get_machine_virt_type", return_value="kvm") 41 | def test_on_install_non_lxc(mock_virt, mock_auditd_install): 42 | ctx = testing.Context(AuditdOperatorCharm) 43 | state = testing.State() 44 | ctx.run(ctx.on.install(), state) 45 | mock_auditd_install.assert_called_once() 46 | 47 | 48 | @patch("charm.AuditdService.install") 49 | @patch("charm.get_machine_virt_type", return_value="lxc") 50 | def test_on_upgrade_lxc(mock_virt, mock_auditd_install): 51 | ctx = testing.Context(AuditdOperatorCharm) 52 | state = testing.State() 53 | with pytest.raises(testing.errors.UncaughtCharmError) as e: 54 | ctx.run(ctx.on.install(), state) 55 | mock_auditd_install.assert_not_called() 56 | assert isinstance(e.value.__cause__, charm.PlatformUnsupportedError) 57 | 58 | 59 | @patch("charm.AuditdService.install") 60 | @patch("charm.get_machine_virt_type", return_value="kvm") 61 | def test_on_upgrade_non_lxc(mock_virt, mock_auditd_install): 62 | ctx = testing.Context(AuditdOperatorCharm) 63 | state = testing.State() 64 | ctx.run(ctx.on.install(), state) 65 | mock_auditd_install.assert_called_once() 66 | 67 | 68 | @pytest.mark.parametrize( 69 | "config", 70 | [ 71 | {"num_logs": -1, "max_log_file": 512}, 72 | {"num_logs": 1000, "max_log_file": 512}, 73 | ], 74 | ) 75 | @patch.object(charm.AuditdOperatorCharm, "_configure_auditd") 76 | def test_configure_charm_invalid_config(_, config): 77 | ctx = testing.Context(AuditdOperatorCharm) 78 | state = testing.State(config=config) 79 | out = ctx.run(ctx.on.config_changed(), state) 80 | assert out.unit_status == testing.BlockedStatus( 81 | "Invalid config. Please check `juju debug-log`." 82 | ) 83 | 84 | 85 | @patch.object( 86 | charm.AuditdOperatorCharm, 87 | "_get_validated_config", 88 | return_value={"num_logs": 2, "max_log_file": 512}, 89 | ) 90 | @patch.object(charm.AuditdOperatorCharm, "_configure_auditd", return_value=False) 91 | def test_configure_charm_failed_configure(_, config): 92 | ctx = testing.Context(AuditdOperatorCharm) 93 | state = testing.State(config=config) 94 | out = ctx.run(ctx.on.config_changed(), state) 95 | assert out.unit_status == testing.BlockedStatus("Failed to configure and restart auditd.") 96 | 97 | 98 | @patch.object(charm.AuditdOperatorCharm, "_configure_auditd", return_value=True) 99 | def test_configure_charm_success(_): 100 | ctx = testing.Context(AuditdOperatorCharm) 101 | state = testing.State(config={"num_logs": 2, "max_log_file": 512}) 102 | out = ctx.run(ctx.on.config_changed(), state) 103 | assert out.unit_status == testing.ActiveStatus() 104 | 105 | 106 | @patch.object(charm.AuditdService, "render_config", return_value="new") 107 | @patch("charm.read_file", return_value="old") 108 | @patch.object(charm.AuditdService, "configure") 109 | @patch.object(charm.AuditdService, "is_active", return_value=True) 110 | def test_configure_auditd_changes_config( 111 | mock_is_active, mock_configure, mock_read_file, mock_render_config 112 | ): 113 | ctx = testing.Context(AuditdOperatorCharm) 114 | state = testing.State(config={"num_logs": 2, "max_log_file": 512}) 115 | out = ctx.run(ctx.on.config_changed(), state) 116 | mock_configure.assert_called_once() 117 | assert out.unit_status == testing.ActiveStatus() 118 | 119 | 120 | @patch.object(charm.AuditdService, "render_config", return_value="same") 121 | @patch("charm.read_file", return_value="same") 122 | @patch.object(charm.AuditdService, "configure") 123 | @patch.object(charm.AuditdService, "is_active", return_value=True) 124 | def test_configure_auditd_no_change( 125 | mock_is_active, mock_configure, mock_read_file, mock_render_config 126 | ): 127 | ctx = testing.Context(AuditdOperatorCharm) 128 | state = testing.State(config={"num_logs": 2, "max_log_file": 512}) 129 | out = ctx.run(ctx.on.config_changed(), state) 130 | mock_configure.assert_not_called() 131 | assert out.unit_status == testing.ActiveStatus() 132 | 133 | 134 | @patch.object(charm.AuditdService, "render_config", return_value="new") 135 | @patch("charm.read_file", return_value="old") 136 | @patch.object( 137 | charm.AuditdService, "configure", side_effect=charm.AuditdServiceRestartError("fail") 138 | ) 139 | @patch.object(charm.AuditdService, "is_active", return_value=True) 140 | def test_configure_auditd_configure_error( 141 | mock_is_active, mock_configure, mock_read_file, mock_render_config 142 | ): 143 | ctx = testing.Context(AuditdOperatorCharm) 144 | state = testing.State(config={"num_logs": 2, "max_log_file": 512}) 145 | out = ctx.run(ctx.on.config_changed(), state) 146 | mock_configure.assert_called_once() 147 | assert out.unit_status == testing.BlockedStatus("Failed to configure and restart auditd.") 148 | 149 | 150 | @patch.object(charm.AuditdService, "render_config", return_value="same") 151 | @patch("charm.read_file", return_value="same") 152 | @patch.object(charm.AuditdService, "is_active", return_value=False) 153 | @patch.object(charm.AuditdService, "restart") 154 | def test_configure_auditd_restart_success( 155 | mock_restart, mock_is_active, mock_read_file, mock_render_config 156 | ): 157 | ctx = testing.Context(AuditdOperatorCharm) 158 | state = testing.State(config={"num_logs": 2, "max_log_file": 512}) 159 | out = ctx.run(ctx.on.config_changed(), state) 160 | mock_restart.assert_called_once() 161 | assert out.unit_status == testing.ActiveStatus() 162 | 163 | 164 | @patch.object(charm.AuditdService, "render_config", return_value="same") 165 | @patch("charm.read_file", return_value="same") 166 | @patch.object(charm.AuditdService, "is_active", return_value=False) 167 | @patch.object(charm.AuditdService, "restart", side_effect=charm.AuditdServiceRestartError("fail")) 168 | def test_configure_auditd_restart_error( 169 | mock_restart, mock_is_active, mock_read_file, mock_render_config 170 | ): 171 | ctx = testing.Context(AuditdOperatorCharm) 172 | state = testing.State(config={"num_logs": 2, "max_log_file": 512}) 173 | out = ctx.run(ctx.on.config_changed(), state) 174 | assert out.unit_status == testing.BlockedStatus("Failed to configure and restart auditd.") 175 | -------------------------------------------------------------------------------- /lib/charms/operator_libs_linux/v1/systemd.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Canonical Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """Abstractions for stopping, starting and managing system services via systemd. 17 | 18 | This library assumes that your charm is running on a platform that uses systemd. E.g., 19 | Centos 7 or later, Ubuntu Xenial (16.04) or later. 20 | 21 | For the most part, we transparently provide an interface to a commonly used selection of 22 | systemd commands, with a few shortcuts baked in. For example, service_pause and 23 | service_resume with run the mask/unmask and enable/disable invocations. 24 | 25 | Example usage: 26 | 27 | ```python 28 | from charms.operator_libs_linux.v0.systemd import service_running, service_reload 29 | 30 | # Start a service 31 | if not service_running("mysql"): 32 | success = service_start("mysql") 33 | 34 | # Attempt to reload a service, restarting if necessary 35 | success = service_reload("nginx", restart_on_failure=True) 36 | ``` 37 | """ 38 | 39 | __all__ = [ # Don't export `_systemctl`. (It's not the intended way of using this lib.) 40 | "SystemdError", 41 | "daemon_reload", 42 | "service_disable", 43 | "service_enable", 44 | "service_failed", 45 | "service_pause", 46 | "service_reload", 47 | "service_restart", 48 | "service_resume", 49 | "service_running", 50 | "service_start", 51 | "service_stop", 52 | ] 53 | 54 | import logging 55 | import subprocess 56 | 57 | logger = logging.getLogger(__name__) 58 | 59 | # The unique Charmhub library identifier, never change it 60 | LIBID = "045b0d179f6b4514a8bb9b48aee9ebaf" 61 | 62 | # Increment this major API version when introducing breaking changes 63 | LIBAPI = 1 64 | 65 | # Increment this PATCH version before using `charmcraft publish-lib` or reset 66 | # to 0 if you are raising the major API version 67 | LIBPATCH = 4 68 | 69 | 70 | class SystemdError(Exception): 71 | """Custom exception for SystemD related errors.""" 72 | 73 | 74 | def _systemctl(*args: str, check: bool = False) -> int: 75 | """Control a system service using systemctl. 76 | 77 | Args: 78 | *args: Arguments to pass to systemctl. 79 | check: Check the output of the systemctl command. Default: False. 80 | 81 | Returns: 82 | Returncode of systemctl command execution. 83 | 84 | Raises: 85 | SystemdError: Raised if calling systemctl returns a non-zero returncode and check is True. 86 | """ 87 | cmd = ["systemctl", *args] 88 | logger.debug(f"Executing command: {cmd}") 89 | try: 90 | proc = subprocess.run( 91 | cmd, 92 | stdout=subprocess.PIPE, 93 | stderr=subprocess.STDOUT, 94 | text=True, 95 | bufsize=1, 96 | encoding="utf-8", 97 | check=check, 98 | ) 99 | logger.debug( 100 | f"Command {cmd} exit code: {proc.returncode}. systemctl output:\n{proc.stdout}" 101 | ) 102 | return proc.returncode 103 | except subprocess.CalledProcessError as e: 104 | raise SystemdError( 105 | f"Command {cmd} failed with returncode {e.returncode}. systemctl output:\n{e.stdout}" 106 | ) 107 | 108 | 109 | def service_running(service_name: str) -> bool: 110 | """Report whether a system service is running. 111 | 112 | Args: 113 | service_name: The name of the service to check. 114 | 115 | Return: 116 | True if service is running/active; False if not. 117 | """ 118 | # If returncode is 0, this means that is service is active. 119 | return _systemctl("--quiet", "is-active", service_name) == 0 120 | 121 | 122 | def service_failed(service_name: str) -> bool: 123 | """Report whether a system service has failed. 124 | 125 | Args: 126 | service_name: The name of the service to check. 127 | 128 | Returns: 129 | True if service is marked as failed; False if not. 130 | """ 131 | # If returncode is 0, this means that the service has failed. 132 | return _systemctl("--quiet", "is-failed", service_name) == 0 133 | 134 | 135 | def service_start(*args: str) -> bool: 136 | """Start a system service. 137 | 138 | Args: 139 | *args: Arguments to pass to `systemctl start` (normally the service name). 140 | 141 | Returns: 142 | On success, this function returns True for historical reasons. 143 | 144 | Raises: 145 | SystemdError: Raised if `systemctl start ...` returns a non-zero returncode. 146 | """ 147 | return _systemctl("start", *args, check=True) == 0 148 | 149 | 150 | def service_stop(*args: str) -> bool: 151 | """Stop a system service. 152 | 153 | Args: 154 | *args: Arguments to pass to `systemctl stop` (normally the service name). 155 | 156 | Returns: 157 | On success, this function returns True for historical reasons. 158 | 159 | Raises: 160 | SystemdError: Raised if `systemctl stop ...` returns a non-zero returncode. 161 | """ 162 | return _systemctl("stop", *args, check=True) == 0 163 | 164 | 165 | def service_restart(*args: str) -> bool: 166 | """Restart a system service. 167 | 168 | Args: 169 | *args: Arguments to pass to `systemctl restart` (normally the service name). 170 | 171 | Returns: 172 | On success, this function returns True for historical reasons. 173 | 174 | Raises: 175 | SystemdError: Raised if `systemctl restart ...` returns a non-zero returncode. 176 | """ 177 | return _systemctl("restart", *args, check=True) == 0 178 | 179 | 180 | def service_enable(*args: str) -> bool: 181 | """Enable a system service. 182 | 183 | Args: 184 | *args: Arguments to pass to `systemctl enable` (normally the service name). 185 | 186 | Returns: 187 | On success, this function returns True for historical reasons. 188 | 189 | Raises: 190 | SystemdError: Raised if `systemctl enable ...` returns a non-zero returncode. 191 | """ 192 | return _systemctl("enable", *args, check=True) == 0 193 | 194 | 195 | def service_disable(*args: str) -> bool: 196 | """Disable a system service. 197 | 198 | Args: 199 | *args: Arguments to pass to `systemctl disable` (normally the service name). 200 | 201 | Returns: 202 | On success, this function returns True for historical reasons. 203 | 204 | Raises: 205 | SystemdError: Raised if `systemctl disable ...` returns a non-zero returncode. 206 | """ 207 | return _systemctl("disable", *args, check=True) == 0 208 | 209 | 210 | def service_reload(service_name: str, restart_on_failure: bool = False) -> bool: 211 | """Reload a system service, optionally falling back to restart if reload fails. 212 | 213 | Args: 214 | service_name: The name of the service to reload. 215 | restart_on_failure: 216 | Boolean indicating whether to fall back to a restart if the reload fails. 217 | 218 | Returns: 219 | On success, this function returns True for historical reasons. 220 | 221 | Raises: 222 | SystemdError: Raised if `systemctl reload|restart ...` returns a non-zero returncode. 223 | """ 224 | try: 225 | return _systemctl("reload", service_name, check=True) == 0 226 | except SystemdError: 227 | if restart_on_failure: 228 | return service_restart(service_name) 229 | else: 230 | raise 231 | 232 | 233 | def service_pause(service_name: str) -> bool: 234 | """Pause a system service. 235 | 236 | Stops the service and prevents the service from starting again at boot. 237 | 238 | Args: 239 | service_name: The name of the service to pause. 240 | 241 | Returns: 242 | On success, this function returns True for historical reasons. 243 | 244 | Raises: 245 | SystemdError: Raised if service is still running after being paused by systemctl. 246 | """ 247 | _systemctl("disable", "--now", service_name) 248 | _systemctl("mask", service_name) 249 | 250 | if service_running(service_name): 251 | raise SystemdError(f"Attempted to pause {service_name!r}, but it is still running.") 252 | 253 | return True 254 | 255 | 256 | def service_resume(service_name: str) -> bool: 257 | """Resume a system service. 258 | 259 | Re-enable starting the service again at boot. Start the service. 260 | 261 | Args: 262 | service_name: The name of the service to resume. 263 | 264 | Returns: 265 | On success, this function returns True for historical reasons. 266 | 267 | Raises: 268 | SystemdError: Raised if service is not running after being resumed by systemctl. 269 | """ 270 | _systemctl("unmask", service_name) 271 | _systemctl("enable", "--now", service_name) 272 | 273 | if not service_running(service_name): 274 | raise SystemdError(f"Attempted to resume {service_name!r}, but it is not running.") 275 | 276 | return True 277 | 278 | 279 | def daemon_reload() -> bool: 280 | """Reload systemd manager configuration. 281 | 282 | Returns: 283 | On success, this function returns True for historical reasons. 284 | 285 | Raises: 286 | SystemdError: Raised if `systemctl daemon-reload` returns a non-zero returncode. 287 | """ 288 | return _systemctl("daemon-reload", check=True) == 0 289 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2025 Canonical Ltd. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /lib/charms/grafana_agent/v0/cos_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Canonical Ltd. 2 | # See LICENSE file for licensing details. 3 | 4 | r"""## Overview. 5 | 6 | This library can be used to manage the cos_agent relation interface: 7 | 8 | - `COSAgentProvider`: Use in machine charms that need to have a workload's metrics 9 | or logs scraped, or forward rule files or dashboards to Prometheus, Loki or Grafana through 10 | the Grafana Agent machine charm. 11 | NOTE: Be sure to add `limit: 1` in your charm for the cos-agent relation. That is the only 12 | way we currently have to prevent two different grafana agent apps deployed on the same VM. 13 | 14 | - `COSAgentConsumer`: Used in the Grafana Agent machine charm to manage the requirer side of 15 | the `cos_agent` interface. 16 | 17 | 18 | ## COSAgentProvider Library Usage 19 | 20 | Grafana Agent machine Charmed Operator interacts with its clients using the cos_agent library. 21 | Charms seeking to send telemetry, must do so using the `COSAgentProvider` object from 22 | this charm library. 23 | 24 | Using the `COSAgentProvider` object only requires instantiating it, 25 | typically in the `__init__` method of your charm (the one which sends telemetry). 26 | 27 | 28 | ```python 29 | def __init__( 30 | self, 31 | charm: CharmType, 32 | relation_name: str = DEFAULT_RELATION_NAME, 33 | metrics_endpoints: Optional[List[_MetricsEndpointDict]] = None, 34 | metrics_rules_dir: str = "./src/prometheus_alert_rules", 35 | logs_rules_dir: str = "./src/loki_alert_rules", 36 | recurse_rules_dirs: bool = False, 37 | log_slots: Optional[List[str]] = None, 38 | dashboard_dirs: Optional[List[str]] = None, 39 | refresh_events: Optional[List] = None, 40 | tracing_protocols: Optional[List[str]] = None, 41 | scrape_configs: Optional[Union[List[Dict], Callable]] = None, 42 | ): 43 | ``` 44 | 45 | ### Parameters 46 | 47 | - `charm`: The instance of the charm that instantiates `COSAgentProvider`, typically `self`. 48 | 49 | - `relation_name`: If your charmed operator uses a relation name other than `cos-agent` to use 50 | the `cos_agent` interface, this is where you have to specify that. 51 | 52 | - `metrics_endpoints`: In this parameter you can specify the metrics endpoints that Grafana Agent 53 | machine Charmed Operator will scrape. The configs of this list will be merged with the configs 54 | from `scrape_configs`. 55 | 56 | - `metrics_rules_dir`: The directory in which the Charmed Operator stores its metrics alert rules 57 | files. 58 | 59 | - `logs_rules_dir`: The directory in which the Charmed Operator stores its logs alert rules files. 60 | 61 | - `recurse_rules_dirs`: This parameters set whether Grafana Agent machine Charmed Operator has to 62 | search alert rules files recursively in the previous two directories or not. 63 | 64 | - `log_slots`: Snap slots to connect to for scraping logs in the form ["snap-name:slot", ...]. 65 | 66 | - `dashboard_dirs`: List of directories where the dashboards are stored in the Charmed Operator. 67 | 68 | - `refresh_events`: List of events on which to refresh relation data. 69 | 70 | - `tracing_protocols`: List of requested tracing protocols that the charm requires to send traces. 71 | 72 | - `scrape_configs`: List of standard scrape_configs dicts or a callable that returns the list in 73 | case the configs need to be generated dynamically. The contents of this list will be merged 74 | with the configs from `metrics_endpoints`. 75 | 76 | 77 | ### Example 1 - Minimal instrumentation: 78 | 79 | In order to use this object the following should be in the `charm.py` file. 80 | 81 | ```python 82 | from charms.grafana_agent.v0.cos_agent import COSAgentProvider 83 | ... 84 | class TelemetryProviderCharm(CharmBase): 85 | def __init__(self, *args): 86 | ... 87 | self._grafana_agent = COSAgentProvider(self) 88 | ``` 89 | 90 | ### Example 2 - Full instrumentation: 91 | 92 | In order to use this object the following should be in the `charm.py` file. 93 | 94 | ```python 95 | from charms.grafana_agent.v0.cos_agent import COSAgentProvider 96 | ... 97 | class TelemetryProviderCharm(CharmBase): 98 | def __init__(self, *args): 99 | ... 100 | self._grafana_agent = COSAgentProvider( 101 | self, 102 | relation_name="custom-cos-agent", 103 | metrics_endpoints=[ 104 | # specify "path" and "port" to scrape from localhost 105 | {"path": "/metrics", "port": 9000}, 106 | {"path": "/metrics", "port": 9001}, 107 | {"path": "/metrics", "port": 9002}, 108 | ], 109 | metrics_rules_dir="./src/alert_rules/prometheus", 110 | logs_rules_dir="./src/alert_rules/loki", 111 | recursive_rules_dir=True, 112 | log_slots=["my-app:slot"], 113 | dashboard_dirs=["./src/dashboards_1", "./src/dashboards_2"], 114 | refresh_events=["update-status", "upgrade-charm"], 115 | tracing_protocols=["otlp_http", "otlp_grpc"], 116 | scrape_configs=[ 117 | { 118 | "job_name": "custom_job", 119 | "metrics_path": "/metrics", 120 | "authorization": {"credentials": "bearer-token"}, 121 | "static_configs": [ 122 | { 123 | "targets": ["localhost:9003"]}, 124 | "labels": {"key": "value"}, 125 | }, 126 | ], 127 | }, 128 | ] 129 | ) 130 | ``` 131 | 132 | ### Example 3 - Dynamic scrape configs generation: 133 | 134 | Pass a function to the `scrape_configs` to decouple the generation of the configs 135 | from the instantiation of the COSAgentProvider object. 136 | 137 | ```python 138 | from charms.grafana_agent.v0.cos_agent import COSAgentProvider 139 | ... 140 | 141 | class TelemetryProviderCharm(CharmBase): 142 | def generate_scrape_configs(self): 143 | return [ 144 | { 145 | "job_name": "custom", 146 | "metrics_path": "/metrics", 147 | "static_configs": [{"targets": ["localhost:9000"]}], 148 | }, 149 | ] 150 | 151 | def __init__(self, *args): 152 | ... 153 | self._grafana_agent = COSAgentProvider( 154 | self, 155 | scrape_configs=self.generate_scrape_configs, 156 | ) 157 | ``` 158 | 159 | ## COSAgentConsumer Library Usage 160 | 161 | This object may be used by any Charmed Operator which gathers telemetry data by 162 | implementing the consumer side of the `cos_agent` interface. 163 | For instance Grafana Agent machine Charmed Operator. 164 | 165 | For this purpose the charm needs to instantiate the `COSAgentConsumer` object with one mandatory 166 | and two optional arguments. 167 | 168 | ### Parameters 169 | 170 | - `charm`: A reference to the parent (Grafana Agent machine) charm. 171 | 172 | - `relation_name`: The name of the relation that the charm uses to interact 173 | with its clients that provides telemetry data using the `COSAgentProvider` object. 174 | 175 | If provided, this relation name must match a provided relation in metadata.yaml with the 176 | `cos_agent` interface. 177 | The default value of this argument is "cos-agent". 178 | 179 | - `refresh_events`: List of events on which to refresh relation data. 180 | 181 | 182 | ### Example 1 - Minimal instrumentation: 183 | 184 | In order to use this object the following should be in the `charm.py` file. 185 | 186 | ```python 187 | from charms.grafana_agent.v0.cos_agent import COSAgentConsumer 188 | ... 189 | class GrafanaAgentMachineCharm(GrafanaAgentCharm) 190 | def __init__(self, *args): 191 | ... 192 | self._cos = COSAgentRequirer(self) 193 | ``` 194 | 195 | 196 | ### Example 2 - Full instrumentation: 197 | 198 | In order to use this object the following should be in the `charm.py` file. 199 | 200 | ```python 201 | from charms.grafana_agent.v0.cos_agent import COSAgentConsumer 202 | ... 203 | class GrafanaAgentMachineCharm(GrafanaAgentCharm) 204 | def __init__(self, *args): 205 | ... 206 | self._cos = COSAgentRequirer( 207 | self, 208 | relation_name="cos-agent-consumer", 209 | refresh_events=["update-status", "upgrade-charm"], 210 | ) 211 | ``` 212 | """ 213 | 214 | import enum 215 | import json 216 | import logging 217 | import socket 218 | from collections import namedtuple 219 | from itertools import chain 220 | from pathlib import Path 221 | from typing import ( 222 | TYPE_CHECKING, 223 | Any, 224 | Callable, 225 | ClassVar, 226 | Dict, 227 | List, 228 | Literal, 229 | MutableMapping, 230 | Optional, 231 | Set, 232 | Tuple, 233 | Union, 234 | ) 235 | 236 | import pydantic 237 | from cosl import DashboardPath40UID, JujuTopology, LZMABase64 238 | from cosl.rules import AlertRules, generic_alert_groups 239 | from ops.charm import RelationChangedEvent 240 | from ops.framework import EventBase, EventSource, Object, ObjectEvents 241 | from ops.model import ModelError, Relation 242 | from ops.testing import CharmType 243 | 244 | if TYPE_CHECKING: 245 | try: 246 | from typing import TypedDict 247 | 248 | class _MetricsEndpointDict(TypedDict): 249 | path: str 250 | port: int 251 | 252 | except ModuleNotFoundError: 253 | _MetricsEndpointDict = Dict # pyright: ignore 254 | 255 | LIBID = "dc15fa84cef84ce58155fb84f6c6213a" 256 | LIBAPI = 0 257 | LIBPATCH = 21 258 | 259 | PYDEPS = ["cosl >= 0.0.50", "pydantic"] 260 | 261 | DEFAULT_RELATION_NAME = "cos-agent" 262 | DEFAULT_PEER_RELATION_NAME = "peers" 263 | DEFAULT_SCRAPE_CONFIG = { 264 | "static_configs": [{"targets": ["localhost:80"]}], 265 | "metrics_path": "/metrics", 266 | } 267 | 268 | logger = logging.getLogger(__name__) 269 | SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") 270 | 271 | # Note: MutableMapping is imported from the typing module and not collections.abc 272 | # because subscripting collections.abc.MutableMapping was added in python 3.9, but 273 | # most of our charms are based on 20.04, which has python 3.8. 274 | 275 | _RawDatabag = MutableMapping[str, str] 276 | 277 | 278 | class TransportProtocolType(str, enum.Enum): 279 | """Receiver Type.""" 280 | 281 | http = "http" 282 | grpc = "grpc" 283 | 284 | 285 | receiver_protocol_to_transport_protocol = { 286 | "zipkin": TransportProtocolType.http, 287 | "kafka": TransportProtocolType.http, 288 | "tempo_http": TransportProtocolType.http, 289 | "tempo_grpc": TransportProtocolType.grpc, 290 | "otlp_grpc": TransportProtocolType.grpc, 291 | "otlp_http": TransportProtocolType.http, 292 | "jaeger_thrift_http": TransportProtocolType.http, 293 | } 294 | 295 | _tracing_receivers_ports = { 296 | # OTLP receiver: see 297 | # https://github.com/open-telemetry/opentelemetry-collector/tree/v0.96.0/receiver/otlpreceiver 298 | "otlp_http": 4318, 299 | "otlp_grpc": 4317, 300 | # Jaeger receiver: see 301 | # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/jaegerreceiver 302 | "jaeger_grpc": 14250, 303 | "jaeger_thrift_http": 14268, 304 | # Zipkin receiver: see 305 | # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/zipkinreceiver 306 | "zipkin": 9411, 307 | } 308 | 309 | ReceiverProtocol = Literal["otlp_grpc", "otlp_http", "zipkin", "jaeger_thrift_http", "jaeger_grpc"] 310 | 311 | 312 | class TracingError(Exception): 313 | """Base class for custom errors raised by tracing.""" 314 | 315 | 316 | class NotReadyError(TracingError): 317 | """Raised by the provider wrapper if a requirer hasn't published the required data (yet).""" 318 | 319 | 320 | class ProtocolNotFoundError(TracingError): 321 | """Raised if the user doesn't receive an endpoint for a protocol it requested.""" 322 | 323 | 324 | class ProtocolNotRequestedError(ProtocolNotFoundError): 325 | """Raised if the user attempts to obtain an endpoint for a protocol it did not request.""" 326 | 327 | 328 | class DataValidationError(TracingError): 329 | """Raised when data validation fails on IPU relation data.""" 330 | 331 | 332 | class AmbiguousRelationUsageError(TracingError): 333 | """Raised when one wrongly assumes that there can only be one relation on an endpoint.""" 334 | 335 | 336 | # TODO we want to eventually use `DatabagModel` from cosl but it likely needs a move to common package first 337 | if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore 338 | 339 | class DatabagModel(pydantic.BaseModel): # type: ignore 340 | """Base databag model.""" 341 | 342 | class Config: 343 | """Pydantic config.""" 344 | 345 | # ignore any extra fields in the databag 346 | extra = "ignore" 347 | """Ignore any extra fields in the databag.""" 348 | allow_population_by_field_name = True 349 | """Allow instantiating this class by field name (instead of forcing alias).""" 350 | 351 | _NEST_UNDER = None 352 | 353 | @classmethod 354 | def load(cls, databag: MutableMapping): 355 | """Load this model from a Juju databag.""" 356 | if cls._NEST_UNDER: 357 | return cls.parse_obj(json.loads(databag[cls._NEST_UNDER])) 358 | 359 | try: 360 | data = { 361 | k: json.loads(v) 362 | for k, v in databag.items() 363 | # Don't attempt to parse model-external values 364 | if k in {f.alias for f in cls.__fields__.values()} 365 | } 366 | except json.JSONDecodeError as e: 367 | msg = f"invalid databag contents: expecting json. {databag}" 368 | logger.error(msg) 369 | raise DataValidationError(msg) from e 370 | 371 | try: 372 | return cls.parse_raw(json.dumps(data)) # type: ignore 373 | except pydantic.ValidationError as e: 374 | msg = f"failed to validate databag: {databag}" 375 | logger.debug(msg, exc_info=True) 376 | raise DataValidationError(msg) from e 377 | 378 | def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): 379 | """Write the contents of this model to Juju databag. 380 | 381 | :param databag: the databag to write the data to. 382 | :param clear: ensure the databag is cleared before writing it. 383 | """ 384 | if clear and databag: 385 | databag.clear() 386 | 387 | if databag is None: 388 | databag = {} 389 | 390 | if self._NEST_UNDER: 391 | databag[self._NEST_UNDER] = self.json(by_alias=True) 392 | return databag 393 | 394 | dct = self.dict() 395 | for key, field in self.__fields__.items(): # type: ignore 396 | value = dct[key] 397 | databag[field.alias or key] = json.dumps(value) 398 | 399 | return databag 400 | 401 | else: 402 | from pydantic import ConfigDict 403 | 404 | class DatabagModel(pydantic.BaseModel): 405 | """Base databag model.""" 406 | 407 | model_config = ConfigDict( 408 | # ignore any extra fields in the databag 409 | extra="ignore", 410 | # Allow instantiating this class by field name (instead of forcing alias). 411 | populate_by_name=True, 412 | # Custom config key: whether to nest the whole datastructure (as json) 413 | # under a field or spread it out at the toplevel. 414 | _NEST_UNDER=None, # type: ignore 415 | arbitrary_types_allowed=True, 416 | ) 417 | """Pydantic config.""" 418 | 419 | @classmethod 420 | def load(cls, databag: MutableMapping): 421 | """Load this model from a Juju databag.""" 422 | nest_under = cls.model_config.get("_NEST_UNDER") # type: ignore 423 | if nest_under: 424 | return cls.model_validate(json.loads(databag[nest_under])) # type: ignore 425 | 426 | try: 427 | data = { 428 | k: json.loads(v) 429 | for k, v in databag.items() 430 | # Don't attempt to parse model-external values 431 | if k in {(f.alias or n) for n, f in cls.__fields__.items()} 432 | } 433 | except json.JSONDecodeError as e: 434 | msg = f"invalid databag contents: expecting json. {databag}" 435 | logger.error(msg) 436 | raise DataValidationError(msg) from e 437 | 438 | try: 439 | return cls.model_validate_json(json.dumps(data)) # type: ignore 440 | except pydantic.ValidationError as e: 441 | msg = f"failed to validate databag: {databag}" 442 | logger.debug(msg, exc_info=True) 443 | raise DataValidationError(msg) from e 444 | 445 | def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): 446 | """Write the contents of this model to Juju databag. 447 | 448 | :param databag: the databag to write the data to. 449 | :param clear: ensure the databag is cleared before writing it. 450 | """ 451 | if clear and databag: 452 | databag.clear() 453 | 454 | if databag is None: 455 | databag = {} 456 | nest_under = self.model_config.get("_NEST_UNDER") 457 | if nest_under: 458 | databag[nest_under] = self.model_dump_json( # type: ignore 459 | by_alias=True, 460 | # skip keys whose values are default 461 | exclude_defaults=True, 462 | ) 463 | return databag 464 | 465 | dct = self.model_dump() # type: ignore 466 | for key, field in self.model_fields.items(): # type: ignore 467 | value = dct[key] 468 | if value == field.default: 469 | continue 470 | databag[field.alias or key] = json.dumps(value) 471 | 472 | return databag 473 | 474 | 475 | class CosAgentProviderUnitData(DatabagModel): # type: ignore 476 | """Unit databag model for `cos-agent` relation.""" 477 | 478 | # The following entries are the same for all units of the same principal. 479 | # Note that the same grafana agent subordinate may be related to several apps. 480 | # this needs to make its way to the gagent leader 481 | metrics_alert_rules: dict 482 | log_alert_rules: dict 483 | dashboards: List[str] 484 | # subordinate is no longer used but we should keep it until we bump the library to ensure 485 | # we don't break compatibility. 486 | subordinate: Optional[bool] = None 487 | 488 | # The following entries may vary across units of the same principal app. 489 | # this data does not need to be forwarded to the gagent leader 490 | metrics_scrape_jobs: List[Dict] 491 | log_slots: List[str] 492 | 493 | # Requested tracing protocols. 494 | tracing_protocols: Optional[List[str]] = None 495 | 496 | # when this whole datastructure is dumped into a databag, it will be nested under this key. 497 | # while not strictly necessary (we could have it 'flattened out' into the databag), 498 | # this simplifies working with the model. 499 | KEY: ClassVar[str] = "config" 500 | 501 | 502 | class CosAgentPeersUnitData(DatabagModel): # type: ignore 503 | """Unit databag model for `peers` cos-agent machine charm peer relation.""" 504 | 505 | # We need the principal unit name and relation metadata to be able to render identifiers 506 | # (e.g. topology) on the leader side, after all the data moves into peer data (the grafana 507 | # agent leader can only see its own principal, because it is a subordinate charm). 508 | unit_name: str 509 | relation_id: str 510 | relation_name: str 511 | 512 | # The only data that is forwarded to the leader is data that needs to go into the app databags 513 | # of the outgoing o11y relations. 514 | metrics_alert_rules: Optional[dict] 515 | log_alert_rules: Optional[dict] 516 | dashboards: Optional[List[str]] 517 | 518 | # when this whole datastructure is dumped into a databag, it will be nested under this key. 519 | # while not strictly necessary (we could have it 'flattened out' into the databag), 520 | # this simplifies working with the model. 521 | KEY: ClassVar[str] = "config" 522 | 523 | @property 524 | def app_name(self) -> str: 525 | """Parse out the app name from the unit name. 526 | 527 | TODO: Switch to using `model_post_init` when pydantic v2 is released? 528 | https://github.com/pydantic/pydantic/issues/1729#issuecomment-1300576214 529 | """ 530 | return self.unit_name.split("/")[0] 531 | 532 | 533 | if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore 534 | 535 | class ProtocolType(pydantic.BaseModel): # type: ignore 536 | """Protocol Type.""" 537 | 538 | class Config: 539 | """Pydantic config.""" 540 | 541 | use_enum_values = True 542 | """Allow serializing enum values.""" 543 | 544 | name: str = pydantic.Field( 545 | ..., 546 | description="Receiver protocol name. What protocols are supported (and what they are called) " 547 | "may differ per provider.", 548 | examples=["otlp_grpc", "otlp_http", "tempo_http"], 549 | ) 550 | 551 | type: TransportProtocolType = pydantic.Field( 552 | ..., 553 | description="The transport protocol used by this receiver.", 554 | examples=["http", "grpc"], 555 | ) 556 | 557 | else: 558 | 559 | class ProtocolType(pydantic.BaseModel): 560 | """Protocol Type.""" 561 | 562 | model_config = pydantic.ConfigDict( 563 | # Allow serializing enum values. 564 | use_enum_values=True 565 | ) 566 | """Pydantic config.""" 567 | 568 | name: str = pydantic.Field( 569 | ..., 570 | description="Receiver protocol name. What protocols are supported (and what they are called) " 571 | "may differ per provider.", 572 | examples=["otlp_grpc", "otlp_http", "tempo_http"], 573 | ) 574 | 575 | type: TransportProtocolType = pydantic.Field( 576 | ..., 577 | description="The transport protocol used by this receiver.", 578 | examples=["http", "grpc"], 579 | ) 580 | 581 | 582 | class Receiver(pydantic.BaseModel): 583 | """Specification of an active receiver.""" 584 | 585 | protocol: ProtocolType = pydantic.Field(..., description="Receiver protocol name and type.") 586 | url: Optional[str] = pydantic.Field( 587 | ..., 588 | description="""URL at which the receiver is reachable. If there's an ingress, it would be the external URL. 589 | Otherwise, it would be the service's fqdn or internal IP. 590 | If the protocol type is grpc, the url will not contain a scheme.""", 591 | examples=[ 592 | "http://traefik_address:2331", 593 | "https://traefik_address:2331", 594 | "http://tempo_public_ip:2331", 595 | "https://tempo_public_ip:2331", 596 | "tempo_public_ip:2331", 597 | ], 598 | ) 599 | 600 | 601 | class CosAgentRequirerUnitData(DatabagModel): # type: ignore 602 | """Application databag model for the COS-agent requirer.""" 603 | 604 | receivers: List[Receiver] = pydantic.Field( 605 | ..., 606 | description="List of all receivers enabled on the tracing provider.", 607 | ) 608 | 609 | 610 | class COSAgentProvider(Object): 611 | """Integration endpoint wrapper for the provider side of the cos_agent interface.""" 612 | 613 | def __init__( 614 | self, 615 | charm: CharmType, 616 | relation_name: str = DEFAULT_RELATION_NAME, 617 | metrics_endpoints: Optional[List["_MetricsEndpointDict"]] = None, 618 | metrics_rules_dir: str = "./src/prometheus_alert_rules", 619 | logs_rules_dir: str = "./src/loki_alert_rules", 620 | recurse_rules_dirs: bool = False, 621 | log_slots: Optional[List[str]] = None, 622 | dashboard_dirs: Optional[List[str]] = None, 623 | refresh_events: Optional[List] = None, 624 | tracing_protocols: Optional[List[str]] = None, 625 | *, 626 | scrape_configs: Optional[Union[List[dict], Callable]] = None, 627 | ): 628 | """Create a COSAgentProvider instance. 629 | 630 | Args: 631 | charm: The `CharmBase` instance that is instantiating this object. 632 | relation_name: The name of the relation to communicate over. 633 | metrics_endpoints: List of endpoints in the form [{"path": path, "port": port}, ...]. 634 | This argument is a simplified form of the `scrape_configs`. 635 | The contents of this list will be merged with the contents of `scrape_configs`. 636 | metrics_rules_dir: Directory where the metrics rules are stored. 637 | logs_rules_dir: Directory where the logs rules are stored. 638 | recurse_rules_dirs: Whether to recurse into rule paths. 639 | log_slots: Snap slots to connect to for scraping logs 640 | in the form ["snap-name:slot", ...]. 641 | dashboard_dirs: Directory where the dashboards are stored. 642 | refresh_events: List of events on which to refresh relation data. 643 | tracing_protocols: List of protocols that the charm will be using for sending traces. 644 | scrape_configs: List of standard scrape_configs dicts or a callable 645 | that returns the list in case the configs need to be generated dynamically. 646 | The contents of this list will be merged with the contents of `metrics_endpoints`. 647 | """ 648 | super().__init__(charm, relation_name) 649 | dashboard_dirs = dashboard_dirs or ["./src/grafana_dashboards"] 650 | 651 | self._charm = charm 652 | self._relation_name = relation_name 653 | self._metrics_endpoints = metrics_endpoints or [] 654 | self._scrape_configs = scrape_configs or [] 655 | self._metrics_rules = metrics_rules_dir 656 | self._logs_rules = logs_rules_dir 657 | self._recursive = recurse_rules_dirs 658 | self._log_slots = log_slots or [] 659 | self._dashboard_dirs = dashboard_dirs 660 | self._refresh_events = refresh_events or [self._charm.on.config_changed] 661 | self._tracing_protocols = tracing_protocols 662 | self._is_single_endpoint = charm.meta.relations[relation_name].limit == 1 663 | 664 | events = self._charm.on[relation_name] 665 | self.framework.observe(events.relation_joined, self._on_refresh) 666 | self.framework.observe(events.relation_changed, self._on_refresh) 667 | for event in self._refresh_events: 668 | self.framework.observe(event, self._on_refresh) 669 | 670 | def _on_refresh(self, event): 671 | """Trigger the class to update relation data.""" 672 | relations = self._charm.model.relations[self._relation_name] 673 | 674 | for relation in relations: 675 | # Before a principal is related to the grafana-agent subordinate, we'd get 676 | # ModelError: ERROR cannot read relation settings: unit "zk/2": settings not found 677 | # Add a guard to make sure it doesn't happen. 678 | if relation.data and self._charm.unit in relation.data: 679 | # Subordinate relations can communicate only over unit data. 680 | try: 681 | data = CosAgentProviderUnitData( 682 | metrics_alert_rules=self._metrics_alert_rules, 683 | log_alert_rules=self._log_alert_rules, 684 | dashboards=self._dashboards, 685 | metrics_scrape_jobs=self._scrape_jobs, 686 | log_slots=self._log_slots, 687 | tracing_protocols=self._tracing_protocols, 688 | ) 689 | relation.data[self._charm.unit][data.KEY] = data.json() 690 | except ( 691 | pydantic.ValidationError, 692 | json.decoder.JSONDecodeError, 693 | ) as e: 694 | logger.error("Invalid relation data provided: %s", e) 695 | 696 | @property 697 | def _scrape_jobs(self) -> List[Dict]: 698 | """Return a prometheus_scrape-like data structure for jobs. 699 | 700 | https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config 701 | """ 702 | if callable(self._scrape_configs): 703 | scrape_configs = self._scrape_configs() 704 | else: 705 | # Create a copy of the user scrape_configs, since we will mutate this object 706 | scrape_configs = self._scrape_configs.copy() 707 | 708 | # Convert "metrics_endpoints" to standard scrape_configs, and add them in 709 | for endpoint in self._metrics_endpoints: 710 | scrape_configs.append( 711 | { 712 | "metrics_path": endpoint["path"], 713 | "static_configs": [{"targets": [f"localhost:{endpoint['port']}"]}], 714 | } 715 | ) 716 | 717 | scrape_configs = scrape_configs or [DEFAULT_SCRAPE_CONFIG] 718 | 719 | # Augment job name to include the app name and a unique id (index) 720 | for idx, scrape_config in enumerate(scrape_configs): 721 | scrape_config["job_name"] = "_".join( 722 | [self._charm.app.name, str(idx), scrape_config.get("job_name", "default")] 723 | ) 724 | 725 | return scrape_configs 726 | 727 | @property 728 | def _metrics_alert_rules(self) -> Dict: 729 | """Use (for now) the prometheus_scrape AlertRules to initialize this.""" 730 | alert_rules = AlertRules( 731 | query_type="promql", topology=JujuTopology.from_charm(self._charm) 732 | ) 733 | alert_rules.add_path(self._metrics_rules, recursive=self._recursive) 734 | alert_rules.add( 735 | generic_alert_groups.application_rules, 736 | group_name_prefix=JujuTopology.from_charm(self._charm).identifier, 737 | ) 738 | return alert_rules.as_dict() 739 | 740 | @property 741 | def _log_alert_rules(self) -> Dict: 742 | """Use (for now) the loki_push_api AlertRules to initialize this.""" 743 | alert_rules = AlertRules(query_type="logql", topology=JujuTopology.from_charm(self._charm)) 744 | alert_rules.add_path(self._logs_rules, recursive=self._recursive) 745 | return alert_rules.as_dict() 746 | 747 | @property 748 | def _dashboards(self) -> List[str]: 749 | dashboards: List[str] = [] 750 | for d in self._dashboard_dirs: 751 | for path in Path(d).glob("*"): 752 | with open(path, "rt") as fp: 753 | dashboard = json.load(fp) 754 | rel_path = str( 755 | path.relative_to(self._charm.charm_dir) if path.is_absolute() else path 756 | ) 757 | # COSAgentProvider is somewhat analogous to GrafanaDashboardProvider. We need to overwrite the uid here 758 | # because there is currently no other way to communicate the dashboard path separately. 759 | # https://github.com/canonical/grafana-k8s-operator/pull/363 760 | dashboard["uid"] = DashboardPath40UID.generate(self._charm.meta.name, rel_path) 761 | 762 | # Add tags 763 | tags: List[str] = dashboard.get("tags", []) 764 | if not any(tag.startswith("charm: ") for tag in tags): 765 | tags.append(f"charm: {self._charm.meta.name}") 766 | dashboard["tags"] = tags 767 | 768 | dashboards.append(LZMABase64.compress(json.dumps(dashboard))) 769 | return dashboards 770 | 771 | @property 772 | def relations(self) -> List[Relation]: 773 | """The tracing relations associated with this endpoint.""" 774 | return self._charm.model.relations[self._relation_name] 775 | 776 | @property 777 | def _relation(self) -> Optional[Relation]: 778 | """If this wraps a single endpoint, the relation bound to it, if any.""" 779 | if not self._is_single_endpoint: 780 | objname = type(self).__name__ 781 | raise AmbiguousRelationUsageError( 782 | f"This {objname} wraps a {self._relation_name} endpoint that has " 783 | "limit != 1. We can't determine what relation, of the possibly many, you are " 784 | f"referring to. Please pass a relation instance while calling {objname}, " 785 | "or set limit=1 in the charm metadata." 786 | ) 787 | relations = self.relations 788 | return relations[0] if relations else None 789 | 790 | def is_ready(self, relation: Optional[Relation] = None): 791 | """Is this endpoint ready?""" 792 | relation = relation or self._relation 793 | if not relation: 794 | logger.debug(f"no relation on {self._relation_name!r}: tracing not ready") 795 | return False 796 | if relation.data is None: 797 | logger.error(f"relation data is None for {relation}") 798 | return False 799 | if not relation.app: 800 | logger.error(f"{relation} event received but there is no relation.app") 801 | return False 802 | try: 803 | unit = next(iter(relation.units), None) 804 | if not unit: 805 | return False 806 | databag = dict(relation.data[unit]) 807 | CosAgentRequirerUnitData.load(databag) 808 | 809 | except (json.JSONDecodeError, pydantic.ValidationError, DataValidationError): 810 | logger.info(f"failed validating relation data for {relation}") 811 | return False 812 | return True 813 | 814 | def get_all_endpoints( 815 | self, relation: Optional[Relation] = None 816 | ) -> Optional[CosAgentRequirerUnitData]: 817 | """Unmarshalled relation data.""" 818 | relation = relation or self._relation 819 | if not relation or not self.is_ready(relation): 820 | return None 821 | unit = next(iter(relation.units), None) 822 | if not unit: 823 | return None 824 | return CosAgentRequirerUnitData.load(relation.data[unit]) # type: ignore 825 | 826 | def _get_tracing_endpoint( 827 | self, relation: Optional[Relation], protocol: ReceiverProtocol 828 | ) -> str: 829 | """Return a tracing endpoint URL if it is available or raise a ProtocolNotFoundError.""" 830 | unit_data = self.get_all_endpoints(relation) 831 | if not unit_data: 832 | # we didn't find the protocol because the remote end didn't publish any data yet 833 | # it might also mean that grafana-agent doesn't have a relation to the tracing backend 834 | raise ProtocolNotFoundError(protocol) 835 | receivers: List[Receiver] = [i for i in unit_data.receivers if i.protocol.name == protocol] 836 | if not receivers: 837 | # we didn't find the protocol because grafana-agent didn't return us the protocol that we requested 838 | # the caller might want to verify that we did indeed request this protocol 839 | raise ProtocolNotFoundError(protocol) 840 | if len(receivers) > 1: 841 | logger.warning( 842 | f"too many receivers with protocol={protocol!r}; using first one. Found: {receivers}" 843 | ) 844 | 845 | receiver = receivers[0] 846 | if not receiver.url: 847 | # grafana-agent isn't connected to the tracing backend yet 848 | raise ProtocolNotFoundError(protocol) 849 | return receiver.url 850 | 851 | def get_tracing_endpoint( 852 | self, protocol: ReceiverProtocol, relation: Optional[Relation] = None 853 | ) -> str: 854 | """Receiver endpoint for the given protocol. 855 | 856 | It could happen that this function gets called before the provider publishes the endpoints. 857 | In such a scenario, if a non-leader unit calls this function, a permission denied exception will be raised due to 858 | restricted access. To prevent this, this function needs to be guarded by the `is_ready` check. 859 | 860 | Raises: 861 | ProtocolNotRequestedError: 862 | If the charm unit is the leader unit and attempts to obtain an endpoint for a protocol it did not request. 863 | ProtocolNotFoundError: 864 | If the charm attempts to obtain an endpoint when grafana-agent isn't related to a tracing backend. 865 | """ 866 | try: 867 | return self._get_tracing_endpoint(relation or self._relation, protocol=protocol) 868 | except ProtocolNotFoundError: 869 | # let's see if we didn't find it because we didn't request the endpoint 870 | requested_protocols = set() 871 | relations = [relation] if relation else self.relations 872 | for relation in relations: 873 | try: 874 | databag = CosAgentProviderUnitData.load(relation.data[self._charm.unit]) 875 | except DataValidationError: 876 | continue 877 | 878 | if databag.tracing_protocols: 879 | requested_protocols.update(databag.tracing_protocols) 880 | 881 | if protocol not in requested_protocols: 882 | raise ProtocolNotRequestedError(protocol, relation) 883 | 884 | raise 885 | 886 | 887 | class COSAgentDataChanged(EventBase): 888 | """Event emitted by `COSAgentRequirer` when relation data changes.""" 889 | 890 | 891 | class COSAgentValidationError(EventBase): 892 | """Event emitted by `COSAgentRequirer` when there is an error in the relation data.""" 893 | 894 | def __init__(self, handle, message: str = ""): 895 | super().__init__(handle) 896 | self.message = message 897 | 898 | def snapshot(self) -> Dict: 899 | """Save COSAgentValidationError source information.""" 900 | return {"message": self.message} 901 | 902 | def restore(self, snapshot): 903 | """Restore COSAgentValidationError source information.""" 904 | self.message = snapshot["message"] 905 | 906 | 907 | class COSAgentRequirerEvents(ObjectEvents): 908 | """`COSAgentRequirer` events.""" 909 | 910 | data_changed = EventSource(COSAgentDataChanged) 911 | validation_error = EventSource(COSAgentValidationError) 912 | 913 | 914 | class COSAgentRequirer(Object): 915 | """Integration endpoint wrapper for the Requirer side of the cos_agent interface.""" 916 | 917 | on = COSAgentRequirerEvents() # pyright: ignore 918 | 919 | def __init__( 920 | self, 921 | charm: CharmType, 922 | *, 923 | relation_name: str = DEFAULT_RELATION_NAME, 924 | peer_relation_name: str = DEFAULT_PEER_RELATION_NAME, 925 | refresh_events: Optional[List[str]] = None, 926 | ): 927 | """Create a COSAgentRequirer instance. 928 | 929 | Args: 930 | charm: The `CharmBase` instance that is instantiating this object. 931 | relation_name: The name of the relation to communicate over. 932 | peer_relation_name: The name of the peer relation to communicate over. 933 | refresh_events: List of events on which to refresh relation data. 934 | """ 935 | super().__init__(charm, relation_name) 936 | self._charm = charm 937 | self._relation_name = relation_name 938 | self._peer_relation_name = peer_relation_name 939 | self._refresh_events = refresh_events or [self._charm.on.config_changed] 940 | 941 | events = self._charm.on[relation_name] 942 | self.framework.observe( 943 | events.relation_joined, self._on_relation_data_changed 944 | ) # TODO: do we need this? 945 | self.framework.observe(events.relation_changed, self._on_relation_data_changed) 946 | self.framework.observe(events.relation_departed, self._on_relation_departed) 947 | 948 | for event in self._refresh_events: 949 | self.framework.observe(event, self.trigger_refresh) # pyright: ignore 950 | 951 | # Peer relation events 952 | # A peer relation is needed as it is the only mechanism for exchanging data across 953 | # subordinate units. 954 | # self.framework.observe( 955 | # self.on[self._peer_relation_name].relation_joined, self._on_peer_relation_joined 956 | # ) 957 | peer_events = self._charm.on[peer_relation_name] 958 | self.framework.observe(peer_events.relation_changed, self._on_peer_relation_changed) 959 | 960 | @property 961 | def peer_relation(self) -> Optional["Relation"]: 962 | """Helper function for obtaining the peer relation object. 963 | 964 | Returns: peer relation object 965 | (NOTE: would return None if called too early, e.g. during install). 966 | """ 967 | return self.model.get_relation(self._peer_relation_name) 968 | 969 | def _on_peer_relation_changed(self, _): 970 | # Peer data is used for forwarding data from principal units to the grafana agent 971 | # subordinate leader, for updating the app data of the outgoing o11y relations. 972 | if self._charm.unit.is_leader(): 973 | self.on.data_changed.emit() # pyright: ignore 974 | 975 | def _on_relation_departed(self, event): 976 | """Remove provider's (principal's) alert rules and dashboards from peer data when the cos-agent relation to the principal is removed.""" 977 | if not self.peer_relation: 978 | event.defer() 979 | return 980 | # empty the departing unit's alert rules and dashboards from peer data 981 | data = CosAgentPeersUnitData( 982 | unit_name=event.unit.name, 983 | relation_id=str(event.relation.id), 984 | relation_name=event.relation.name, 985 | metrics_alert_rules={}, 986 | log_alert_rules={}, 987 | dashboards=[], 988 | ) 989 | self.peer_relation.data[self._charm.unit][ 990 | f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" 991 | ] = data.json() 992 | 993 | self.on.data_changed.emit() # pyright: ignore 994 | 995 | def _on_relation_data_changed(self, event: RelationChangedEvent): 996 | # Peer data is the only means of communication between subordinate units. 997 | if not self.peer_relation: 998 | event.defer() 999 | return 1000 | 1001 | cos_agent_relation = event.relation 1002 | if not event.unit or not cos_agent_relation.data.get(event.unit): 1003 | return 1004 | principal_unit = event.unit 1005 | 1006 | # Coherence check 1007 | units = cos_agent_relation.units 1008 | if len(units) > 1: 1009 | # should never happen 1010 | raise ValueError( 1011 | f"unexpected error: subordinate relation {cos_agent_relation} " 1012 | f"should have exactly one unit" 1013 | ) 1014 | 1015 | if not (raw := cos_agent_relation.data[principal_unit].get(CosAgentProviderUnitData.KEY)): 1016 | return 1017 | 1018 | if not (provider_data := self._validated_provider_data(raw)): 1019 | return 1020 | 1021 | # write enabled receivers to cos-agent relation 1022 | try: 1023 | self.update_tracing_receivers() 1024 | except ModelError: 1025 | raise 1026 | 1027 | # Copy data from the cos_agent relation to the peer relation, so the leader could 1028 | # follow up. 1029 | # Save the originating unit name, so it could be used for topology later on by the leader. 1030 | data = CosAgentPeersUnitData( # peer relation databag model 1031 | unit_name=event.unit.name, 1032 | relation_id=str(event.relation.id), 1033 | relation_name=event.relation.name, 1034 | metrics_alert_rules=provider_data.metrics_alert_rules, 1035 | log_alert_rules=provider_data.log_alert_rules, 1036 | dashboards=provider_data.dashboards, 1037 | ) 1038 | self.peer_relation.data[self._charm.unit][ 1039 | f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" 1040 | ] = data.json() 1041 | 1042 | # We can't easily tell if the data that was changed is limited to only the data 1043 | # that goes into peer relation (in which case, if this is not a leader unit, we wouldn't 1044 | # need to emit `on.data_changed`), so we're emitting `on.data_changed` either way. 1045 | self.on.data_changed.emit() # pyright: ignore 1046 | 1047 | def update_tracing_receivers(self): 1048 | """Updates the list of exposed tracing receivers in all relations.""" 1049 | try: 1050 | for relation in self._charm.model.relations[self._relation_name]: 1051 | CosAgentRequirerUnitData( 1052 | receivers=[ 1053 | Receiver( 1054 | # if tracing isn't ready, we don't want the wrong receiver URLs present in the databag. 1055 | # however, because of the backwards compatibility requirements, we need to still provide 1056 | # the protocols list so that the charm with older cos_agent version doesn't error its hooks. 1057 | # before this change was added, the charm with old cos_agent version threw exceptions with 1058 | # connections to grafana-agent timing out. After the change, the charm will fail validating 1059 | # databag contents (as it expects a string in URL) but that won't cause any errors as 1060 | # tracing endpoints are the only content in the grafana-agent's side of the databag. 1061 | url=f"{self._get_tracing_receiver_url(protocol)}" 1062 | if self._charm.tracing.is_ready() # type: ignore 1063 | else None, 1064 | protocol=ProtocolType( 1065 | name=protocol, 1066 | type=receiver_protocol_to_transport_protocol[protocol], 1067 | ), 1068 | ) 1069 | for protocol in self.requested_tracing_protocols() 1070 | ], 1071 | ).dump(relation.data[self._charm.unit]) 1072 | 1073 | except ModelError as e: 1074 | # args are bytes 1075 | msg = e.args[0] 1076 | if isinstance(msg, bytes): 1077 | if msg.startswith( 1078 | b"ERROR cannot read relation application settings: permission denied" 1079 | ): 1080 | logger.error( 1081 | f"encountered error {e} while attempting to update_relation_data." 1082 | f"The relation must be gone." 1083 | ) 1084 | return 1085 | raise 1086 | 1087 | def _validated_provider_data(self, raw) -> Optional[CosAgentProviderUnitData]: 1088 | try: 1089 | return CosAgentProviderUnitData(**json.loads(raw)) 1090 | except (pydantic.ValidationError, json.decoder.JSONDecodeError) as e: 1091 | self.on.validation_error.emit(message=str(e)) # pyright: ignore 1092 | return None 1093 | 1094 | def trigger_refresh(self, _): 1095 | """Trigger a refresh of relation data.""" 1096 | # FIXME: Figure out what we should do here 1097 | self.on.data_changed.emit() # pyright: ignore 1098 | 1099 | def _get_requested_protocols(self, relation: Relation): 1100 | # Coherence check 1101 | units = relation.units 1102 | if len(units) > 1: 1103 | # should never happen 1104 | raise ValueError( 1105 | f"unexpected error: subordinate relation {relation} should have exactly one unit" 1106 | ) 1107 | 1108 | unit = next(iter(units), None) 1109 | 1110 | if not unit: 1111 | return None 1112 | 1113 | if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): 1114 | return None 1115 | 1116 | if not (provider_data := self._validated_provider_data(raw)): 1117 | return None 1118 | 1119 | return provider_data.tracing_protocols 1120 | 1121 | def requested_tracing_protocols(self): 1122 | """All receiver protocols that have been requested by our related apps.""" 1123 | requested_protocols = set() 1124 | for relation in self._charm.model.relations[self._relation_name]: 1125 | try: 1126 | protocols = self._get_requested_protocols(relation) 1127 | except NotReadyError: 1128 | continue 1129 | if protocols: 1130 | requested_protocols.update(protocols) 1131 | return requested_protocols 1132 | 1133 | def _get_tracing_receiver_url(self, protocol: str): 1134 | scheme = "http" 1135 | try: 1136 | if self._charm.cert.enabled: # type: ignore 1137 | scheme = "https" 1138 | # not only Grafana Agent can implement cos_agent. If the charm doesn't have the `cert` attribute 1139 | # using our cert_handler, it won't have the `enabled` parameter. In this case, we pass and assume http. 1140 | except AttributeError: 1141 | pass 1142 | # the assumption is that a subordinate charm will always be accessible to its principal charm under its fqdn 1143 | if receiver_protocol_to_transport_protocol[protocol] == TransportProtocolType.grpc: 1144 | return f"{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" 1145 | return f"{scheme}://{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" 1146 | 1147 | @property 1148 | def _remote_data(self) -> List[Tuple[CosAgentProviderUnitData, JujuTopology]]: 1149 | """Return a list of remote data from each of the related units. 1150 | 1151 | Assumes that the relation is of type subordinate. 1152 | Relies on the fact that, for subordinate relations, the only remote unit visible to 1153 | *this unit* is the principal unit that this unit is attached to. 1154 | """ 1155 | all_data = [] 1156 | 1157 | for relation in self._charm.model.relations[self._relation_name]: 1158 | if not relation.units: 1159 | continue 1160 | unit = next(iter(relation.units)) 1161 | if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): 1162 | continue 1163 | if not (provider_data := self._validated_provider_data(raw)): 1164 | continue 1165 | 1166 | topology = JujuTopology( 1167 | model=self._charm.model.name, 1168 | model_uuid=self._charm.model.uuid, 1169 | application=unit.app.name, 1170 | unit=unit.name, 1171 | ) 1172 | 1173 | all_data.append((provider_data, topology)) 1174 | 1175 | return all_data 1176 | 1177 | def _gather_peer_data(self) -> List[CosAgentPeersUnitData]: 1178 | """Collect data from the peers. 1179 | 1180 | Returns a trimmed-down list of CosAgentPeersUnitData. 1181 | """ 1182 | relation = self.peer_relation 1183 | 1184 | # Ensure that whatever context we're running this in, we take the necessary precautions: 1185 | if not relation or not relation.data or not relation.app: 1186 | return [] 1187 | 1188 | # Iterate over all peer unit data and only collect every principal once. 1189 | peer_data: List[CosAgentPeersUnitData] = [] 1190 | app_names: Set[str] = set() 1191 | 1192 | for unit in chain((self._charm.unit,), relation.units): 1193 | if not relation.data.get(unit): 1194 | continue 1195 | 1196 | for unit_name in relation.data.get(unit): # pyright: ignore 1197 | if not unit_name.startswith(CosAgentPeersUnitData.KEY): 1198 | continue 1199 | raw = relation.data[unit].get(unit_name) 1200 | if raw is None: 1201 | continue 1202 | data = CosAgentPeersUnitData(**json.loads(raw)) 1203 | # Have we already seen this principal app? 1204 | if (app_name := data.app_name) in app_names: 1205 | continue 1206 | peer_data.append(data) 1207 | app_names.add(app_name) 1208 | 1209 | return peer_data 1210 | 1211 | @property 1212 | def metrics_alerts(self) -> Dict[str, Any]: 1213 | """Fetch metrics alerts.""" 1214 | alert_rules = {} 1215 | 1216 | seen_apps: List[str] = [] 1217 | for data in self._gather_peer_data(): 1218 | if rules := data.metrics_alert_rules: 1219 | app_name = data.app_name 1220 | if app_name in seen_apps: 1221 | continue # dedup! 1222 | seen_apps.append(app_name) 1223 | # This is only used for naming the file, so be as specific as we can be 1224 | identifier = JujuTopology( 1225 | model=self._charm.model.name, 1226 | model_uuid=self._charm.model.uuid, 1227 | application=app_name, 1228 | # For the topology unit, we could use `data.principal_unit_name`, but that unit 1229 | # name may not be very stable: `_gather_peer_data` de-duplicates by app name so 1230 | # the exact unit name that turns up first in the iterator may vary from time to 1231 | # time. So using the grafana-agent unit name instead. 1232 | unit=self._charm.unit.name, 1233 | ).identifier 1234 | 1235 | alert_rules[identifier] = rules 1236 | 1237 | return alert_rules 1238 | 1239 | @property 1240 | def metrics_jobs(self) -> List[Dict]: 1241 | """Parse the relation data contents and extract the metrics jobs.""" 1242 | scrape_jobs = [] 1243 | for data, topology in self._remote_data: 1244 | for job in data.metrics_scrape_jobs: 1245 | # In #220, relation schema changed from a simplified dict to the standard 1246 | # `scrape_configs`. 1247 | # This is to ensure backwards compatibility with Providers older than v0.5. 1248 | if "path" in job and "port" in job and "job_name" in job: 1249 | job = { 1250 | "job_name": job["job_name"], 1251 | "metrics_path": job["path"], 1252 | "static_configs": [{"targets": [f"localhost:{job['port']}"]}], 1253 | # We include insecure_skip_verify because we are always scraping localhost. 1254 | # Even if we have the certs for the scrape targets, we'd rather specify the scrape 1255 | # jobs with localhost rather than the SAN DNS the cert was issued for. 1256 | "tls_config": {"insecure_skip_verify": True}, 1257 | } 1258 | 1259 | # Apply labels to the scrape jobs 1260 | for static_config in job.get("static_configs", []): 1261 | topo_as_dict = topology.as_dict(excluded_keys=["charm_name"]) 1262 | static_config["labels"] = { 1263 | # Be sure to keep labels from static_config 1264 | **static_config.get("labels", {}), 1265 | # TODO: We should add a new method in juju_topology.py 1266 | # that like `as_dict` method, returns the keys with juju_ prefix 1267 | # https://github.com/canonical/cos-lib/issues/18 1268 | **{ 1269 | "juju_{}".format(key): value 1270 | for key, value in topo_as_dict.items() 1271 | if value 1272 | }, 1273 | } 1274 | 1275 | scrape_jobs.append(job) 1276 | 1277 | return scrape_jobs 1278 | 1279 | @property 1280 | def snap_log_endpoints(self) -> List[SnapEndpoint]: 1281 | """Fetch logging endpoints exposed by related snaps.""" 1282 | endpoints = [] 1283 | endpoints_with_topology = self.snap_log_endpoints_with_topology 1284 | for endpoint, _ in endpoints_with_topology: 1285 | endpoints.append(endpoint) 1286 | 1287 | return endpoints 1288 | 1289 | @property 1290 | def snap_log_endpoints_with_topology(self) -> List[Tuple[SnapEndpoint, JujuTopology]]: 1291 | """Fetch logging endpoints and charm topology for each related snap.""" 1292 | plugs = [] 1293 | for data, topology in self._remote_data: 1294 | targets = data.log_slots 1295 | if targets: 1296 | for target in targets: 1297 | if target in plugs: 1298 | logger.warning( 1299 | f"plug {target} already listed. " 1300 | "The same snap is being passed from multiple " 1301 | "endpoints; this should not happen." 1302 | ) 1303 | else: 1304 | plugs.append((target, topology)) 1305 | 1306 | endpoints = [] 1307 | for plug, topology in plugs: 1308 | if ":" not in plug: 1309 | logger.error(f"invalid plug definition received: {plug}. Ignoring...") 1310 | else: 1311 | endpoint = SnapEndpoint(*plug.split(":")) 1312 | endpoints.append((endpoint, topology)) 1313 | 1314 | return endpoints 1315 | 1316 | @property 1317 | def logs_alerts(self) -> Dict[str, Any]: 1318 | """Fetch log alerts.""" 1319 | alert_rules = {} 1320 | seen_apps: List[str] = [] 1321 | 1322 | for data in self._gather_peer_data(): 1323 | if rules := data.log_alert_rules: 1324 | # This is only used for naming the file, so be as specific as we can be 1325 | app_name = data.app_name 1326 | if app_name in seen_apps: 1327 | continue # dedup! 1328 | seen_apps.append(app_name) 1329 | 1330 | identifier = JujuTopology( 1331 | model=self._charm.model.name, 1332 | model_uuid=self._charm.model.uuid, 1333 | application=app_name, 1334 | # For the topology unit, we could use `data.unit_name`, but that unit 1335 | # name may not be very stable: `_gather_peer_data` de-duplicates by app name so 1336 | # the exact unit name that turns up first in the iterator may vary from time to 1337 | # time. So using the grafana-agent unit name instead. 1338 | unit=self._charm.unit.name, 1339 | ).identifier 1340 | 1341 | alert_rules[identifier] = rules 1342 | 1343 | return alert_rules 1344 | 1345 | @property 1346 | def dashboards(self) -> List[Dict[str, str]]: 1347 | """Fetch dashboards as encoded content. 1348 | 1349 | Dashboards are assumed not to vary across units of the same primary. 1350 | """ 1351 | dashboards: List[Dict[str, Any]] = [] 1352 | 1353 | seen_apps: List[str] = [] 1354 | for data in self._gather_peer_data(): 1355 | app_name = data.app_name 1356 | if app_name in seen_apps: 1357 | continue # dedup! 1358 | seen_apps.append(app_name) 1359 | 1360 | for encoded_dashboard in data.dashboards or (): 1361 | content = json.loads(LZMABase64.decompress(encoded_dashboard)) 1362 | 1363 | title = content.get("title", "no_title") 1364 | 1365 | dashboards.append( 1366 | { 1367 | "relation_id": data.relation_id, 1368 | # We have the remote charm name - use it for the identifier 1369 | "charm": f"{data.relation_name}-{app_name}", 1370 | "content": content, 1371 | "title": title, 1372 | } 1373 | ) 1374 | 1375 | return dashboards 1376 | 1377 | 1378 | def charm_tracing_config( 1379 | endpoint_requirer: COSAgentProvider, cert_path: Optional[Union[Path, str]] 1380 | ) -> Tuple[Optional[str], Optional[str]]: 1381 | """Utility function to determine the charm_tracing config you will likely want. 1382 | 1383 | If no endpoint is provided: 1384 | disable charm tracing. 1385 | If https endpoint is provided but cert_path is not found on disk: 1386 | disable charm tracing. 1387 | If https endpoint is provided and cert_path is None: 1388 | raise TracingError 1389 | Else: 1390 | proceed with charm tracing (with or without tls, as appropriate) 1391 | 1392 | Usage: 1393 | >>> from lib.charms.tempo_coordinator_k8s.v0.charm_tracing import trace_charm 1394 | >>> from lib.charms.tempo_coordinator_k8s.v0.tracing import charm_tracing_config 1395 | >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") 1396 | >>> class MyCharm(...): 1397 | >>> _cert_path = "/path/to/cert/on/charm/container.crt" 1398 | >>> def __init__(self, ...): 1399 | >>> self.tracing = TracingEndpointRequirer(...) 1400 | >>> self.my_endpoint, self.cert_path = charm_tracing_config( 1401 | ... self.tracing, self._cert_path) 1402 | """ 1403 | if not endpoint_requirer.is_ready(): 1404 | return None, None 1405 | 1406 | try: 1407 | endpoint = endpoint_requirer.get_tracing_endpoint("otlp_http") 1408 | except ProtocolNotFoundError: 1409 | logger.warn( 1410 | "Endpoint for tracing wasn't provided as tracing backend isn't ready yet. If grafana-agent isn't connected to a tracing backend, integrate it. Otherwise this issue should resolve itself in a few events." 1411 | ) 1412 | return None, None 1413 | 1414 | if not endpoint: 1415 | return None, None 1416 | 1417 | is_https = endpoint.startswith("https://") 1418 | 1419 | if is_https: 1420 | if cert_path is None: 1421 | raise TracingError("Cannot send traces to an https endpoint without a certificate.") 1422 | if not Path(cert_path).exists(): 1423 | # if endpoint is https BUT we don't have a server_cert yet: 1424 | # disable charm tracing until we do to prevent tls errors 1425 | return None, None 1426 | return endpoint, str(cert_path) 1427 | return endpoint, None 1428 | -------------------------------------------------------------------------------- /lib/charms/operator_libs_linux/v0/apt.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Canonical Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Abstractions for the system's Debian/Ubuntu package information and repositories. 16 | 17 | This module contains abstractions and wrappers around Debian/Ubuntu-style repositories and 18 | packages, in order to easily provide an idiomatic and Pythonic mechanism for adding packages and/or 19 | repositories to systems for use in machine charms. 20 | 21 | A sane default configuration is attainable through nothing more than instantiation of the 22 | appropriate classes. `DebianPackage` objects provide information about the architecture, version, 23 | name, and status of a package. 24 | 25 | `DebianPackage` will try to look up a package either from `dpkg -L` or from `apt-cache` when 26 | provided with a string indicating the package name. If it cannot be located, `PackageNotFoundError` 27 | will be returned, as `apt` and `dpkg` otherwise return `100` for all errors, and a meaningful error 28 | message if the package is not known is desirable. 29 | 30 | To install packages with convenience methods: 31 | 32 | ```python 33 | try: 34 | # Run `apt-get update` 35 | apt.update() 36 | apt.add_package("zsh") 37 | apt.add_package(["vim", "htop", "wget"]) 38 | except PackageError as e: 39 | logger.error("could not install package. Reason: %s", e.message) 40 | ```` 41 | 42 | The convenience methods don't raise `PackageNotFoundError`. If any packages aren't found in 43 | the cache, `apt.add_package` raises `PackageError` with a message 'Failed to install 44 | packages: foo, bar'. 45 | 46 | To find details of a specific package: 47 | 48 | ```python 49 | try: 50 | vim = apt.DebianPackage.from_system("vim") 51 | 52 | # To find from the apt cache only 53 | # apt.DebianPackage.from_apt_cache("vim") 54 | 55 | # To find from installed packages only 56 | # apt.DebianPackage.from_installed_package("vim") 57 | 58 | vim.ensure(PackageState.Latest) 59 | logger.info("updated vim to version: %s", vim.fullversion) 60 | except PackageNotFoundError: 61 | logger.error("a specified package not found in package cache or on system") 62 | except PackageError as e: 63 | logger.error("could not install package. Reason: %s", e.message) 64 | ``` 65 | 66 | 67 | `RepositoryMapping` will return a dict-like object containing enabled system repositories 68 | and their properties (available groups, baseuri. gpg key). This class can add, disable, or 69 | manipulate repositories. Items can be retrieved as `DebianRepository` objects. 70 | 71 | In order to add a new repository with explicit details for fields, a new `DebianRepository` 72 | can be added to `RepositoryMapping` 73 | 74 | `RepositoryMapping` provides an abstraction around the existing repositories on the system, 75 | and can be accessed and iterated over like any `Mapping` object, to retrieve values by key, 76 | iterate, or perform other operations. 77 | 78 | Keys are constructed as `{repo_type}-{}-{release}` in order to uniquely identify a repository. 79 | 80 | Repositories can be added with explicit values through a Python constructor. 81 | 82 | Example: 83 | ```python 84 | repositories = apt.RepositoryMapping() 85 | 86 | if "deb-example.com-focal" not in repositories: 87 | repositories.add(DebianRepository(enabled=True, repotype="deb", 88 | uri="https://example.com", release="focal", groups=["universe"])) 89 | ``` 90 | 91 | Alternatively, any valid `sources.list` line may be used to construct a new 92 | `DebianRepository`. 93 | 94 | Example: 95 | ```python 96 | repositories = apt.RepositoryMapping() 97 | 98 | if "deb-us.archive.ubuntu.com-xenial" not in repositories: 99 | line = "deb http://us.archive.ubuntu.com/ubuntu xenial main restricted" 100 | repo = DebianRepository.from_repo_line(line) 101 | repositories.add(repo) 102 | ``` 103 | 104 | Dependencies: 105 | Note that this module requires `opentelemetry-api`, which is already included into 106 | your charm's virtual environment via `ops >= 2.21`. 107 | """ 108 | 109 | from __future__ import annotations 110 | 111 | import fileinput 112 | import glob 113 | import logging 114 | import os 115 | import re 116 | import subprocess 117 | import typing 118 | from enum import Enum 119 | from subprocess import PIPE, CalledProcessError, check_output 120 | from typing import Any, Iterable, Iterator, Literal, Mapping 121 | from urllib.parse import urlparse 122 | 123 | import opentelemetry.trace 124 | 125 | logger = logging.getLogger(__name__) 126 | tracer = opentelemetry.trace.get_tracer(__name__) 127 | 128 | # The unique Charmhub library identifier, never change it 129 | LIBID = "7c3dbc9c2ad44a47bd6fcb25caa270e5" 130 | 131 | # Increment this major API version when introducing breaking changes 132 | LIBAPI = 0 133 | 134 | # Increment this PATCH version before using `charmcraft publish-lib` or reset 135 | # to 0 if you are raising the major API version 136 | LIBPATCH = 19 137 | 138 | PYDEPS = ["opentelemetry-api"] 139 | 140 | 141 | VALID_SOURCE_TYPES = ("deb", "deb-src") 142 | OPTIONS_MATCHER = re.compile(r"\[.*?\]") 143 | _GPG_KEY_DIR = "/etc/apt/trusted.gpg.d/" 144 | 145 | 146 | class Error(Exception): 147 | """Base class of most errors raised by this library.""" 148 | 149 | def __repr__(self): 150 | """Represent the Error.""" 151 | return f"<{type(self).__module__}.{type(self).__name__} {self.args}>" 152 | 153 | @property 154 | def name(self): 155 | """Return a string representation of the model plus class.""" 156 | return f"<{type(self).__module__}.{type(self).__name__}>" 157 | 158 | @property 159 | def message(self): 160 | """Return the message passed as an argument.""" 161 | return self.args[0] 162 | 163 | 164 | class PackageError(Error): 165 | """Raised when there's an error installing or removing a package. 166 | 167 | Additionally, `apt.add_package` raises `PackageError` if any packages aren't found in 168 | the cache. 169 | """ 170 | 171 | 172 | class PackageNotFoundError(Error): 173 | """Raised by `DebianPackage` methods if a requested package is not found.""" 174 | 175 | 176 | class PackageState(Enum): 177 | """A class to represent possible package states.""" 178 | 179 | Present = "present" 180 | Absent = "absent" 181 | Latest = "latest" 182 | Available = "available" 183 | 184 | 185 | class DebianPackage: 186 | """Represents a traditional Debian package and its utility functions. 187 | 188 | `DebianPackage` wraps information and functionality around a known package, whether installed 189 | or available. The version, epoch, name, and architecture can be easily queried and compared 190 | against other `DebianPackage` objects to determine the latest version or to install a specific 191 | version. 192 | 193 | The representation of this object as a string mimics the output from `dpkg` for familiarity. 194 | 195 | Installation and removal of packages is handled through the `state` property or `ensure` 196 | method, with the following options: 197 | 198 | apt.PackageState.Absent 199 | apt.PackageState.Available 200 | apt.PackageState.Present 201 | apt.PackageState.Latest 202 | 203 | When `DebianPackage` is initialized, the state of a given `DebianPackage` object will be set to 204 | `Available`, `Present`, or `Latest`, with `Absent` implemented as a convenience for removal 205 | (though it operates essentially the same as `Available`). 206 | """ 207 | 208 | def __init__( 209 | self, name: str, version: str, epoch: str, arch: str, state: PackageState 210 | ) -> None: 211 | self._name = name 212 | self._arch = arch 213 | self._state = state 214 | self._version = Version(version, epoch) 215 | 216 | def __eq__(self, other: object) -> bool: 217 | """Equality for comparison. 218 | 219 | Args: 220 | other: a `DebianPackage` object for comparison 221 | 222 | Returns: 223 | A boolean reflecting equality 224 | """ 225 | return isinstance(other, self.__class__) and ( 226 | self._name, 227 | self._version.number, 228 | ) == (other._name, other._version.number) 229 | 230 | def __hash__(self): 231 | """Return a hash of this package.""" 232 | return hash((self._name, self._version.number)) 233 | 234 | def __repr__(self): 235 | """Represent the package.""" 236 | return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" 237 | 238 | def __str__(self): 239 | """Return a human-readable representation of the package.""" 240 | return ( 241 | f"<{type(self).__name__}: {self._name}-{self._version}.{self._arch} -- {self._state}>" 242 | ) 243 | 244 | @staticmethod 245 | def _apt( 246 | command: str, 247 | package_names: str | list[str], 248 | optargs: list[str] | None = None, 249 | ) -> None: 250 | """Wrap package management commands for Debian/Ubuntu systems. 251 | 252 | Args: 253 | command: the command given to `apt-get` 254 | package_names: a package name or list of package names to operate on 255 | optargs: an (Optional) list of additional arguments 256 | 257 | Raises: 258 | PackageError if an error is encountered 259 | """ 260 | optargs = optargs if optargs is not None else [] 261 | if isinstance(package_names, str): 262 | package_names = [package_names] 263 | _cmd = ["apt-get", "-y", *optargs, command, *package_names] 264 | try: 265 | env = os.environ.copy() 266 | env["DEBIAN_FRONTEND"] = "noninteractive" 267 | with tracer.start_as_current_span(_cmd[0]) as span: 268 | span.set_attribute("argv", _cmd) 269 | subprocess.run(_cmd, capture_output=True, check=True, text=True, env=env) 270 | except CalledProcessError as e: 271 | raise PackageError( 272 | f"Could not {command} package(s) {package_names}: {e.stderr}" 273 | ) from None 274 | 275 | def _add(self) -> None: 276 | """Add a package to the system.""" 277 | self._apt( 278 | "install", 279 | f"{self.name}={self.version}", 280 | optargs=["--option=Dpkg::Options::=--force-confold"], 281 | ) 282 | 283 | def _remove(self) -> None: 284 | """Remove a package from the system. Implementation-specific.""" 285 | return self._apt("remove", f"{self.name}={self.version}") 286 | 287 | @property 288 | def name(self) -> str: 289 | """Returns the name of the package.""" 290 | return self._name 291 | 292 | def ensure(self, state: PackageState): 293 | """Ensure that a package is in a given state. 294 | 295 | Args: 296 | state: a `PackageState` to reconcile the package to 297 | 298 | Raises: 299 | PackageError from the underlying call to apt 300 | """ 301 | if self._state is not state: 302 | if state not in (PackageState.Present, PackageState.Latest): 303 | self._remove() 304 | else: 305 | self._add() 306 | self._state = state 307 | 308 | @property 309 | def present(self) -> bool: 310 | """Returns whether or not a package is present.""" 311 | return self._state in (PackageState.Present, PackageState.Latest) 312 | 313 | @property 314 | def latest(self) -> bool: 315 | """Returns whether the package is the most recent version.""" 316 | return self._state is PackageState.Latest 317 | 318 | @property 319 | def state(self) -> PackageState: 320 | """Returns the current package state.""" 321 | return self._state 322 | 323 | @state.setter 324 | def state(self, state: PackageState) -> None: 325 | """Set the package state to a given value. 326 | 327 | Args: 328 | state: a `PackageState` to reconcile the package to 329 | 330 | Raises: 331 | PackageError from the underlying call to apt 332 | """ 333 | if state in (PackageState.Latest, PackageState.Present): 334 | self._add() 335 | else: 336 | self._remove() 337 | self._state = state 338 | 339 | @property 340 | def version(self) -> Version: 341 | """Returns the version for a package.""" 342 | return self._version 343 | 344 | @property 345 | def epoch(self) -> str: 346 | """Returns the epoch for a package. May be unset.""" 347 | return self._version.epoch 348 | 349 | @property 350 | def arch(self) -> str: 351 | """Returns the architecture for a package.""" 352 | return self._arch 353 | 354 | @property 355 | def fullversion(self) -> str: 356 | """Returns the name+epoch for a package.""" 357 | return f"{self._version}.{self._arch}" 358 | 359 | @staticmethod 360 | def _get_epoch_from_version(version: str) -> tuple[str, str]: 361 | """Pull the epoch, if any, out of a version string.""" 362 | epoch_matcher = re.compile(r"^((?P\d+):)?(?P.*)") 363 | result = epoch_matcher.search(version) 364 | assert result is not None 365 | matches = result.groupdict() 366 | return matches.get("epoch", ""), matches["version"] 367 | 368 | @classmethod 369 | def from_system( 370 | cls, package: str, version: str | None = "", arch: str | None = "" 371 | ) -> DebianPackage: 372 | """Locates a package, either on the system or known to apt, and serializes the information. 373 | 374 | Args: 375 | package: a string representing the package 376 | version: an optional string if a specific version is requested 377 | arch: an optional architecture, defaulting to `dpkg --print-architecture`. If an 378 | architecture is not specified, this will be used for selection. 379 | 380 | """ 381 | try: 382 | return DebianPackage.from_installed_package(package, version, arch) 383 | except PackageNotFoundError: 384 | logger.debug( 385 | "package '%s' is not currently installed or has the wrong architecture.", package 386 | ) 387 | 388 | # Ok, try `apt-cache ...` 389 | try: 390 | return DebianPackage.from_apt_cache(package, version, arch) 391 | except (PackageNotFoundError, PackageError): 392 | # If we get here, it's not known to the systems. 393 | # This seems unnecessary, but virtually all `apt` commands have a return code of `100`, 394 | # and providing meaningful error messages without this is ugly. 395 | arch_str = f".{arch}" if arch else "" 396 | raise PackageNotFoundError( 397 | f"Package '{package}{arch_str}' " 398 | "could not be found on the system or in the apt cache!" 399 | ) from None 400 | 401 | @classmethod 402 | def from_installed_package( 403 | cls, package: str, version: str | None = "", arch: str | None = "" 404 | ) -> DebianPackage: 405 | """Check whether the package is already installed and return an instance. 406 | 407 | Args: 408 | package: a string representing the package 409 | version: an optional string if a specific version is requested 410 | arch: an optional architecture, defaulting to `dpkg --print-architecture`. 411 | If an architecture is not specified, this will be used for selection. 412 | """ 413 | system_arch = check_output( 414 | ["dpkg", "--print-architecture"], universal_newlines=True 415 | ).strip() 416 | arch = arch if arch else system_arch 417 | 418 | # Regexps are a really terrible way to do this. Thanks dpkg 419 | output = "" 420 | try: 421 | output = check_output(["dpkg", "-l", package], stderr=PIPE, universal_newlines=True) 422 | except CalledProcessError: 423 | raise PackageNotFoundError(f"Package is not installed: {package}") from None 424 | 425 | # Pop off the output from `dpkg -l' because there's no flag to 426 | # omit it` 427 | lines = str(output).splitlines()[5:] 428 | 429 | dpkg_matcher = re.compile( 430 | r""" 431 | ^(?P\w+?)\s+ 432 | (?P.*?)(?P:\w+?)?\s+ 433 | (?P.*?)\s+ 434 | (?P\w+?)\s+ 435 | (?P.*) 436 | """, 437 | re.VERBOSE, 438 | ) 439 | 440 | for line in lines: 441 | result = dpkg_matcher.search(line) 442 | if result is None: 443 | logger.warning("dpkg matcher could not parse line: %s", line) 444 | continue 445 | matches = result.groupdict() 446 | package_status = matches["package_status"] 447 | 448 | if not package_status.endswith("i"): 449 | logger.debug( 450 | "package '%s' in dpkg output but not installed, status: '%s'", 451 | package, 452 | package_status, 453 | ) 454 | break 455 | 456 | epoch, split_version = DebianPackage._get_epoch_from_version(matches["version"]) 457 | pkg = DebianPackage( 458 | name=matches["package_name"], 459 | version=split_version, 460 | epoch=epoch, 461 | arch=matches["arch"], 462 | state=PackageState.Present, 463 | ) 464 | if (pkg.arch == "all" or pkg.arch == arch) and ( 465 | version == "" or str(pkg.version) == version 466 | ): 467 | return pkg 468 | 469 | # If we didn't find it, fail through 470 | raise PackageNotFoundError(f"Package {package}.{arch} is not installed!") 471 | 472 | @classmethod 473 | def from_apt_cache( 474 | cls, package: str, version: str | None = "", arch: str | None = "" 475 | ) -> DebianPackage: 476 | """Check whether the package is already installed and return an instance. 477 | 478 | Args: 479 | package: a string representing the package 480 | version: an optional string if a specific version is requested 481 | arch: an optional architecture, defaulting to `dpkg --print-architecture`. 482 | If an architecture is not specified, this will be used for selection. 483 | """ 484 | cmd = ["dpkg", "--print-architecture"] 485 | with tracer.start_as_current_span(cmd[0]) as span: 486 | span.set_attribute("argv", cmd) 487 | system_arch = check_output(cmd, universal_newlines=True).strip() 488 | arch = arch if arch else system_arch 489 | 490 | # Regexps are a really terrible way to do this. Thanks dpkg 491 | keys = ("Package", "Architecture", "Version") 492 | 493 | cmd = ["apt-cache", "show", package] 494 | try: 495 | with tracer.start_as_current_span(cmd[0]) as span: 496 | span.set_attribute("argv", cmd) 497 | output = check_output(cmd, stderr=PIPE, universal_newlines=True) 498 | except CalledProcessError as e: 499 | raise PackageError(f"Could not list packages in apt-cache: {e.stderr}") from None 500 | 501 | pkg_groups = output.strip().split("\n\n") 502 | keys = ("Package", "Architecture", "Version") 503 | 504 | for pkg_raw in pkg_groups: 505 | lines = str(pkg_raw).splitlines() 506 | vals: dict[str, str] = {} 507 | for line in lines: 508 | if line.startswith(keys): 509 | items = line.split(":", 1) 510 | vals[items[0]] = items[1].strip() 511 | else: 512 | continue 513 | 514 | epoch, split_version = DebianPackage._get_epoch_from_version(vals["Version"]) 515 | pkg = DebianPackage( 516 | name=vals["Package"], 517 | version=split_version, 518 | epoch=epoch, 519 | arch=vals["Architecture"], 520 | state=PackageState.Available, 521 | ) 522 | 523 | if (pkg.arch == "all" or pkg.arch == arch) and ( 524 | version == "" or str(pkg.version) == version 525 | ): 526 | return pkg 527 | 528 | # If we didn't find it, fail through 529 | raise PackageNotFoundError(f"Package {package}.{arch} is not in the apt cache!") 530 | 531 | 532 | class Version: 533 | """An abstraction around package versions. 534 | 535 | This seems like it should be strictly unnecessary, except that `apt_pkg` is not usable inside a 536 | venv, and wedging version comparisons into `DebianPackage` would overcomplicate it. 537 | 538 | This class implements the algorithm found here: 539 | https://www.debian.org/doc/debian-policy/ch-controlfields.html#version 540 | """ 541 | 542 | def __init__(self, version: str, epoch: str): 543 | self._version = version 544 | self._epoch = epoch or "" 545 | 546 | def __repr__(self): 547 | """Represent the package.""" 548 | return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" 549 | 550 | def __str__(self): 551 | """Return human-readable representation of the package.""" 552 | epoch = f"{self._epoch}:" if self._epoch else "" 553 | return f"{epoch}{self._version}" 554 | 555 | @property 556 | def epoch(self): 557 | """Returns the epoch for a package. May be empty.""" 558 | return self._epoch 559 | 560 | @property 561 | def number(self) -> str: 562 | """Returns the version number for a package.""" 563 | return self._version 564 | 565 | def _get_parts(self, version: str) -> tuple[str, str]: 566 | """Separate the version into component upstream and Debian pieces.""" 567 | try: 568 | version.rindex("-") 569 | except ValueError: 570 | # No hyphens means no Debian version 571 | return version, "0" 572 | 573 | upstream, debian = version.rsplit("-", 1) 574 | return upstream, debian 575 | 576 | def _listify(self, revision: str) -> list[str | int]: 577 | """Split a revision string into a list. 578 | 579 | This list is comprised of alternating between strings and numbers, 580 | padded on either end to always be "str, int, str, int..." and 581 | always be of even length. This allows us to trivially implement the 582 | comparison algorithm described. 583 | """ 584 | result: list[str | int] = [] 585 | while revision: 586 | rev_1, remains = self._get_alphas(revision) 587 | rev_2, remains = self._get_digits(remains) 588 | result.extend([rev_1, rev_2]) 589 | revision = remains 590 | return result 591 | 592 | def _get_alphas(self, revision: str) -> tuple[str, str]: 593 | """Return a tuple of the first non-digit characters of a revision.""" 594 | # get the index of the first digit 595 | for i, char in enumerate(revision): 596 | if char.isdigit(): 597 | if i == 0: 598 | return "", revision 599 | return revision[0:i], revision[i:] 600 | # string is entirely alphas 601 | return revision, "" 602 | 603 | def _get_digits(self, revision: str) -> tuple[int, str]: 604 | """Return a tuple of the first integer characters of a revision.""" 605 | # If the string is empty, return (0,'') 606 | if not revision: 607 | return 0, "" 608 | # get the index of the first non-digit 609 | for i, char in enumerate(revision): 610 | if not char.isdigit(): 611 | if i == 0: 612 | return 0, revision 613 | return int(revision[0:i]), revision[i:] 614 | # string is entirely digits 615 | return int(revision), "" 616 | 617 | def _dstringcmp(self, a: str, b: str) -> Literal[-1, 0, 1]: 618 | """Debian package version string section lexical sort algorithm. 619 | 620 | The lexical comparison is a comparison of ASCII values modified so 621 | that all the letters sort earlier than all the non-letters and so that 622 | a tilde sorts before anything, even the end of a part. 623 | """ 624 | if a == b: 625 | return 0 626 | try: 627 | for i, char in enumerate(a): 628 | if char == b[i]: 629 | continue 630 | # "a tilde sorts before anything, even the end of a part" 631 | # (emptyness) 632 | if char == "~": 633 | return -1 634 | if b[i] == "~": 635 | return 1 636 | # "all the letters sort earlier than all the non-letters" 637 | if char.isalpha() and not b[i].isalpha(): 638 | return -1 639 | if not char.isalpha() and b[i].isalpha(): 640 | return 1 641 | # otherwise lexical sort 642 | if ord(char) > ord(b[i]): 643 | return 1 644 | if ord(char) < ord(b[i]): 645 | return -1 646 | except IndexError: 647 | # a is longer than b but otherwise equal, greater unless there are tildes 648 | # FIXME: type checker thinks "char" is possibly unbound as it's a loop variable 649 | # but it won't be since the IndexError can only occur inside the loop 650 | # -- I'd like to refactor away this `try ... except` anyway 651 | if char == "~": # pyright: ignore[reportPossiblyUnboundVariable] 652 | return -1 653 | return 1 654 | # if we get here, a is shorter than b but otherwise equal, so check for tildes... 655 | if b[len(a)] == "~": 656 | return 1 657 | return -1 658 | 659 | def _compare_revision_strings(self, first: str, second: str) -> Literal[-1, 0, 1]: 660 | """Compare two debian revision strings.""" 661 | if first == second: 662 | return 0 663 | 664 | # listify pads results so that we will always be comparing ints to ints 665 | # and strings to strings (at least until we fall off the end of a list) 666 | first_list = self._listify(first) 667 | second_list = self._listify(second) 668 | if first_list == second_list: 669 | return 0 670 | try: 671 | for i, item in enumerate(first_list): 672 | # explicitly raise IndexError if we've fallen off the edge of list2 673 | if i >= len(second_list): 674 | raise IndexError 675 | other = second_list[i] 676 | # if the items are equal, next 677 | if item == other: 678 | continue 679 | # numeric comparison 680 | if isinstance(item, int): 681 | assert isinstance(other, int) 682 | if item > other: 683 | return 1 684 | if item < other: 685 | return -1 686 | else: 687 | # string comparison 688 | assert isinstance(other, str) 689 | return self._dstringcmp(item, other) 690 | except IndexError: 691 | # rev1 is longer than rev2 but otherwise equal, hence greater 692 | # ...except for goddamn tildes 693 | # FIXME: bug?? we return 1 in both cases 694 | # FIXME: first_list[len(second_list)] should be a string 695 | # why are we indexing to 0 twice? 696 | if first_list[len(second_list)][0][0] == "~": # type: ignore 697 | return 1 698 | return 1 699 | # rev1 is shorter than rev2 but otherwise equal, hence lesser 700 | # ...except for goddamn tildes 701 | # FIXME: bug?? we return -1 in both cases 702 | # FIXME: first_list[len(second_list)] should be a string, why are we indexing to 0 twice? 703 | if second_list[len(first_list)][0][0] == "~": # type: ignore 704 | return -1 705 | return -1 706 | 707 | def _compare_version(self, other: Version) -> Literal[-1, 0, 1]: 708 | if (self.number, self.epoch) == (other.number, other.epoch): 709 | return 0 710 | 711 | if self.epoch < other.epoch: 712 | return -1 713 | if self.epoch > other.epoch: 714 | return 1 715 | 716 | # If none of these are true, follow the algorithm 717 | upstream_version, debian_version = self._get_parts(self.number) 718 | other_upstream_version, other_debian_version = self._get_parts(other.number) 719 | 720 | upstream_cmp = self._compare_revision_strings(upstream_version, other_upstream_version) 721 | if upstream_cmp != 0: 722 | return upstream_cmp 723 | 724 | debian_cmp = self._compare_revision_strings(debian_version, other_debian_version) 725 | if debian_cmp != 0: 726 | return debian_cmp 727 | 728 | return 0 729 | 730 | def __lt__(self, other: Version) -> bool: 731 | """Less than magic method impl.""" 732 | return self._compare_version(other) < 0 733 | 734 | def __eq__(self, other: object) -> bool: 735 | """Equality magic method impl.""" 736 | if not isinstance(other, Version): 737 | return False 738 | return self._compare_version(other) == 0 739 | 740 | def __gt__(self, other: Version) -> bool: 741 | """Greater than magic method impl.""" 742 | return self._compare_version(other) > 0 743 | 744 | def __le__(self, other: Version) -> bool: 745 | """Less than or equal to magic method impl.""" 746 | return self.__eq__(other) or self.__lt__(other) 747 | 748 | def __ge__(self, other: Version) -> bool: 749 | """Greater than or equal to magic method impl.""" 750 | return self.__gt__(other) or self.__eq__(other) 751 | 752 | def __ne__(self, other: object) -> bool: 753 | """Not equal to magic method impl.""" 754 | return not self.__eq__(other) 755 | 756 | 757 | @typing.overload 758 | def add_package( 759 | package_names: str, 760 | version: str | None = "", 761 | arch: str | None = "", 762 | update_cache: bool = False, 763 | ) -> DebianPackage: ... 764 | @typing.overload 765 | def add_package( 766 | package_names: list[str], 767 | version: str | None = "", 768 | arch: str | None = "", 769 | update_cache: bool = False, 770 | ) -> DebianPackage | list[DebianPackage]: ... 771 | def add_package( 772 | package_names: str | list[str], 773 | version: str | None = "", 774 | arch: str | None = "", 775 | update_cache: bool = False, 776 | ) -> DebianPackage | list[DebianPackage]: 777 | """Add a package or list of packages to the system. 778 | 779 | Args: 780 | package_names: single package name, or list of package names 781 | name: the name(s) of the package(s) 782 | version: an (Optional) version as a string. Defaults to the latest known 783 | arch: an optional architecture for the package 784 | update_cache: whether or not to run `apt-get update` prior to operating 785 | 786 | Raises: 787 | TypeError if no package name is given, or explicit version is set for multiple packages 788 | PackageError: if packages fail to install, including if any packages aren't found in the 789 | cache 790 | """ 791 | cache_refreshed = False 792 | if update_cache: 793 | update() 794 | cache_refreshed = True 795 | 796 | package_names = [package_names] if isinstance(package_names, str) else package_names 797 | if not package_names: 798 | raise TypeError("Expected at least one package name to add, received zero!") 799 | 800 | if len(package_names) != 1 and version: 801 | raise TypeError( 802 | "Explicit version should not be set if more than one package is being added!" 803 | ) 804 | 805 | succeeded: list[DebianPackage] = [] 806 | retry: list[str] = [] 807 | failed: list[str] = [] 808 | 809 | for p in package_names: 810 | pkg, _ = _add(p, version, arch) 811 | if isinstance(pkg, DebianPackage): 812 | succeeded.append(pkg) 813 | elif cache_refreshed: 814 | logger.warning("failed to locate and install/update '%s'", pkg) 815 | failed.append(p) 816 | else: 817 | logger.warning("failed to locate and install/update '%s', will retry later", pkg) 818 | retry.append(p) 819 | 820 | if retry: 821 | logger.info("updating the apt-cache and retrying installation of failed packages.") 822 | update() 823 | 824 | for p in retry: 825 | pkg, _ = _add(p, version, arch) 826 | if isinstance(pkg, DebianPackage): 827 | succeeded.append(pkg) 828 | else: 829 | failed.append(p) 830 | 831 | if failed: 832 | raise PackageError(f"Failed to install packages: {', '.join(failed)}") 833 | 834 | return succeeded[0] if len(succeeded) == 1 else succeeded 835 | 836 | 837 | def _add( 838 | name: str, 839 | version: str | None = "", 840 | arch: str | None = "", 841 | ) -> tuple[DebianPackage, Literal[True]] | tuple[str, Literal[False]]: 842 | """Add a package to the system. 843 | 844 | Args: 845 | name: the name(s) of the package(s) 846 | version: an (Optional) version as a string. Defaults to the latest known 847 | arch: an optional architecture for the package 848 | 849 | Returns: a tuple of `DebianPackage` if found, or a :str: if it is not, and 850 | a boolean indicating success 851 | """ 852 | try: 853 | pkg = DebianPackage.from_system(name, version, arch) 854 | pkg.ensure(state=PackageState.Present) 855 | return pkg, True 856 | except PackageNotFoundError: 857 | return name, False 858 | 859 | 860 | @typing.overload 861 | def remove_package( 862 | package_names: str, 863 | ) -> DebianPackage: ... 864 | @typing.overload 865 | def remove_package( 866 | package_names: list[str], 867 | ) -> DebianPackage | list[DebianPackage]: ... 868 | def remove_package( 869 | package_names: str | list[str], 870 | ) -> DebianPackage | list[DebianPackage]: 871 | """Remove package(s) from the system. 872 | 873 | Args: 874 | package_names: the name of a package 875 | 876 | Raises: 877 | TypeError: if no packages are provided 878 | """ 879 | packages: list[DebianPackage] = [] 880 | 881 | package_names = [package_names] if isinstance(package_names, str) else package_names 882 | if not package_names: 883 | raise TypeError("Expected at least one package name to add, received zero!") 884 | 885 | for p in package_names: 886 | try: 887 | pkg = DebianPackage.from_installed_package(p) 888 | pkg.ensure(state=PackageState.Absent) 889 | packages.append(pkg) 890 | except PackageNotFoundError: # noqa: PERF203 891 | logger.info("package '%s' was requested for removal, but it was not installed.", p) 892 | 893 | # the list of packages will be empty when no package is removed 894 | logger.debug("packages: '%s'", packages) 895 | return packages[0] if len(packages) == 1 else packages 896 | 897 | 898 | def update() -> None: 899 | """Update the apt cache via `apt-get update`.""" 900 | cmd = ["apt-get", "update", "--error-on=any"] 901 | try: 902 | with tracer.start_as_current_span(cmd[0]) as span: 903 | span.set_attribute("argv", cmd) 904 | subprocess.run(cmd, capture_output=True, check=True) 905 | except CalledProcessError as e: 906 | logger.error( 907 | "%s:\nstdout:\n%s\nstderr:\n%s", 908 | " ".join(cmd), 909 | e.stdout.decode(), 910 | e.stderr.decode(), 911 | ) 912 | raise 913 | 914 | 915 | def import_key(key: str) -> str: 916 | """Import an ASCII Armor key. 917 | 918 | A Radix64 format keyid is also supported for backwards 919 | compatibility. In this case Ubuntu keyserver will be 920 | queried for a key via HTTPS by its keyid. This method 921 | is less preferable because https proxy servers may 922 | require traffic decryption which is equivalent to a 923 | man-in-the-middle attack (a proxy server impersonates 924 | keyserver TLS certificates and has to be explicitly 925 | trusted by the system). 926 | 927 | Args: 928 | key: A GPG key in ASCII armor format, including BEGIN 929 | and END markers or a keyid. 930 | 931 | Returns: 932 | The GPG key filename written. 933 | 934 | Raises: 935 | GPGKeyError if the key could not be imported 936 | """ 937 | key = key.strip() 938 | if "-" in key or "\n" in key: 939 | # Send everything not obviously a keyid to GPG to import, as 940 | # we trust its validation better than our own. eg. handling 941 | # comments before the key. 942 | logger.debug("PGP key found (looks like ASCII Armor format)") 943 | if ( 944 | "-----BEGIN PGP PUBLIC KEY BLOCK-----" in key 945 | and "-----END PGP PUBLIC KEY BLOCK-----" in key 946 | ): 947 | logger.debug("Writing provided PGP key in the binary format") 948 | key_bytes = key.encode("utf-8") 949 | key_name = DebianRepository._get_keyid_by_gpg_key(key_bytes) 950 | key_gpg = DebianRepository._dearmor_gpg_key(key_bytes) 951 | gpg_key_filename = os.path.join(_GPG_KEY_DIR, f"{key_name}.gpg") 952 | DebianRepository._write_apt_gpg_keyfile( 953 | key_name=gpg_key_filename, key_material=key_gpg 954 | ) 955 | return gpg_key_filename 956 | else: 957 | raise GPGKeyError("ASCII armor markers missing from GPG key") 958 | else: 959 | logger.warning( 960 | "PGP key found (looks like Radix64 format). " 961 | "SECURELY importing PGP key from keyserver; " 962 | "full key not provided." 963 | ) 964 | # as of bionic add-apt-repository uses curl with an HTTPS keyserver URL 965 | # to retrieve GPG keys. `apt-key adv` command is deprecated as is 966 | # apt-key in general as noted in its manpage. See lp:1433761 for more 967 | # history. Instead, /etc/apt/trusted.gpg.d is used directly to drop 968 | # gpg 969 | key_asc = DebianRepository._get_key_by_keyid(key) 970 | # write the key in GPG format so that apt-key list shows it 971 | key_gpg = DebianRepository._dearmor_gpg_key(key_asc.encode("utf-8")) 972 | gpg_key_filename = os.path.join(_GPG_KEY_DIR, f"{key}.gpg") 973 | DebianRepository._write_apt_gpg_keyfile(key_name=gpg_key_filename, key_material=key_gpg) 974 | return gpg_key_filename 975 | 976 | 977 | class InvalidSourceError(Error): 978 | """Exceptions for invalid source entries.""" 979 | 980 | 981 | class GPGKeyError(Error): 982 | """Exceptions for GPG keys.""" 983 | 984 | 985 | class DebianRepository: 986 | """An abstraction to represent a repository.""" 987 | 988 | _deb822_stanza: _Deb822Stanza | None = None 989 | """set by Deb822Stanza after creating a DebianRepository""" 990 | 991 | def __init__( 992 | self, 993 | enabled: bool, 994 | repotype: str, 995 | uri: str, 996 | release: str, 997 | groups: list[str], 998 | filename: str = "", 999 | gpg_key_filename: str = "", 1000 | options: dict[str, str] | None = None, 1001 | ): 1002 | self._enabled = enabled 1003 | self._repotype = repotype 1004 | self._uri = uri 1005 | self._release = release 1006 | self._groups = groups 1007 | self._filename = filename 1008 | self._gpg_key_filename = gpg_key_filename 1009 | self._options = options 1010 | 1011 | @property 1012 | def enabled(self): 1013 | """Return whether or not the repository is enabled.""" 1014 | return self._enabled 1015 | 1016 | @property 1017 | def repotype(self): 1018 | """Return whether it is binary or source.""" 1019 | return self._repotype 1020 | 1021 | @property 1022 | def uri(self): 1023 | """Return the URI.""" 1024 | return self._uri 1025 | 1026 | @property 1027 | def release(self): 1028 | """Return which Debian/Ubuntu releases it is valid for.""" 1029 | return self._release 1030 | 1031 | @property 1032 | def groups(self): 1033 | """Return the enabled package groups.""" 1034 | return self._groups 1035 | 1036 | @property 1037 | def filename(self): 1038 | """Returns the filename for a repository.""" 1039 | return self._filename 1040 | 1041 | @filename.setter 1042 | def filename(self, fname: str) -> None: 1043 | """Set the filename used when a repo is written back to disk. 1044 | 1045 | Args: 1046 | fname: a filename to write the repository information to. 1047 | """ 1048 | if not fname.endswith((".list", ".sources")): 1049 | raise InvalidSourceError("apt source filenames should end in .list or .sources!") 1050 | self._filename = fname 1051 | 1052 | @property 1053 | def gpg_key(self): 1054 | """Returns the path to the GPG key for this repository.""" 1055 | if not self._gpg_key_filename and self._deb822_stanza is not None: 1056 | self._gpg_key_filename = self._deb822_stanza.get_gpg_key_filename() 1057 | return self._gpg_key_filename 1058 | 1059 | @property 1060 | def options(self): 1061 | """Returns any additional repo options which are set.""" 1062 | return self._options 1063 | 1064 | def make_options_string(self, include_signed_by: bool = True) -> str: 1065 | """Generate the complete one-line-style options string for a repository. 1066 | 1067 | Combining `gpg_key`, if set (and include_signed_by is True), with any other 1068 | provided options to form the options section of a one-line-style definition. 1069 | """ 1070 | options = self._options if self._options else {} 1071 | if include_signed_by and self.gpg_key: 1072 | options["signed-by"] = self.gpg_key 1073 | if not options: 1074 | return "" 1075 | pairs = (f"{k}={v}" for k, v in sorted(options.items())) 1076 | return "[{}] ".format(" ".join(pairs)) 1077 | 1078 | @staticmethod 1079 | def prefix_from_uri(uri: str) -> str: 1080 | """Get a repo list prefix from the uri, depending on whether a path is set.""" 1081 | uridetails = urlparse(uri) 1082 | path = ( 1083 | uridetails.path.lstrip("/").replace("/", "-") if uridetails.path else uridetails.netloc 1084 | ) 1085 | return f"/etc/apt/sources.list.d/{path}" 1086 | 1087 | @staticmethod 1088 | def from_repo_line(repo_line: str, write_file: bool | None = True) -> DebianRepository: 1089 | """Instantiate a new `DebianRepository` from a `sources.list` entry line. 1090 | 1091 | Args: 1092 | repo_line: a string representing a repository entry 1093 | write_file: boolean to enable writing the new repo to disk. True by default. 1094 | Expect it to result in an add-apt-repository call under the hood, like: 1095 | add-apt-repository --no-update --sourceslist="$repo_line" 1096 | """ 1097 | repo = RepositoryMapping._parse( 1098 | repo_line, 1099 | filename="UserInput", # temp filename 1100 | ) 1101 | repo.filename = repo._make_filename() 1102 | if write_file: 1103 | _add_repository(repo) 1104 | return repo 1105 | 1106 | def _make_filename(self) -> str: 1107 | """Construct a filename from uri and release. 1108 | 1109 | For internal use when a filename isn't set. 1110 | Should match the filename written to by add-apt-repository. 1111 | """ 1112 | return "{}-{}.list".format( 1113 | DebianRepository.prefix_from_uri(self.uri), 1114 | self.release.replace("/", "-"), 1115 | ) 1116 | 1117 | def disable(self) -> None: 1118 | """Remove this repository by disabling it in the source file. 1119 | 1120 | WARNING: This method does NOT alter the `self.enabled` flag. 1121 | 1122 | WARNING: disable is currently not implemented for repositories defined 1123 | by a deb822 stanza. Raises a NotImplementedError in this case. 1124 | """ 1125 | if self._deb822_stanza is not None: 1126 | raise NotImplementedError( 1127 | "Disabling a repository defined by a deb822 format source is not implemented." 1128 | " Please raise an issue if you require this feature." 1129 | ) 1130 | searcher = f"{self.repotype} {self.make_options_string()}{self.uri} {self.release}" 1131 | with tracer.start_as_current_span("disable source") as span: 1132 | span.set_attribute("filename", self._filename) 1133 | with fileinput.input(self._filename, inplace=True) as lines: 1134 | for line in lines: 1135 | if re.match(rf"^{re.escape(searcher)}\s", line): 1136 | print(f"# {line}", end="") 1137 | else: 1138 | print(line, end="") 1139 | 1140 | def import_key(self, key: str) -> None: 1141 | """Import an ASCII Armor key. 1142 | 1143 | A Radix64 format keyid is also supported for backwards 1144 | compatibility. In this case Ubuntu keyserver will be 1145 | queried for a key via HTTPS by its keyid. This method 1146 | is less preferable because https proxy servers may 1147 | require traffic decryption which is equivalent to a 1148 | man-in-the-middle attack (a proxy server impersonates 1149 | keyserver TLS certificates and has to be explicitly 1150 | trusted by the system). 1151 | 1152 | Args: 1153 | key: A GPG key in ASCII armor format, 1154 | including BEGIN and END markers or a keyid. 1155 | 1156 | Raises: 1157 | GPGKeyError if the key could not be imported 1158 | """ 1159 | self._gpg_key_filename = import_key(key) 1160 | 1161 | @staticmethod 1162 | def _get_keyid_by_gpg_key(key_material: bytes) -> str: 1163 | """Get a GPG key fingerprint by GPG key material. 1164 | 1165 | Gets a GPG key fingerprint (40-digit, 160-bit) by the ASCII armor-encoded 1166 | or binary GPG key material. Can be used, for example, to generate file 1167 | names for keys passed via charm options. 1168 | """ 1169 | # Use the same gpg command for both Xenial and Bionic 1170 | cmd = ["gpg", "--with-colons", "--with-fingerprint"] 1171 | with tracer.start_as_current_span(cmd[0]) as span: 1172 | span.set_attribute("argv", cmd) 1173 | ps = subprocess.run(cmd, capture_output=True, input=key_material) 1174 | out, err = ps.stdout.decode(), ps.stderr.decode() 1175 | if "gpg: no valid OpenPGP data found." in err: 1176 | raise GPGKeyError("Invalid GPG key material provided") 1177 | # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) 1178 | result = re.search(r"^fpr:{9}([0-9A-F]{40}):$", out, re.MULTILINE) 1179 | assert result is not None 1180 | return result.group(1) 1181 | 1182 | @staticmethod 1183 | def _get_key_by_keyid(keyid: str) -> str: 1184 | """Get a key via HTTPS from the Ubuntu keyserver. 1185 | 1186 | Different key ID formats are supported by SKS keyservers (the longer ones 1187 | are more secure, see "dead beef attack" and https://evil32.com/). Since 1188 | HTTPS is used, if SSLBump-like HTTPS proxies are in place, they will 1189 | impersonate keyserver.ubuntu.com and generate a certificate with 1190 | keyserver.ubuntu.com in the CN field or in SubjAltName fields of a 1191 | certificate. If such proxy behavior is expected it is necessary to add the 1192 | CA certificate chain containing the intermediate CA of the SSLBump proxy to 1193 | every machine that this code runs on via ca-certs cloud-init directive (via 1194 | cloudinit-userdata model-config) or via other means (such as through a 1195 | custom charm option). Also note that DNS resolution for the hostname in a 1196 | URL is done at a proxy server - not at the client side. 1197 | 8-digit (32 bit) key ID 1198 | https://keyserver.ubuntu.com/pks/lookup?search=0x4652B4E6 1199 | 16-digit (64 bit) key ID 1200 | https://keyserver.ubuntu.com/pks/lookup?search=0x6E85A86E4652B4E6 1201 | 40-digit key ID: 1202 | https://keyserver.ubuntu.com/pks/lookup?search=0x35F77D63B5CEC106C577ED856E85A86E4652B4E6 1203 | 1204 | Args: 1205 | keyid: An 8, 16 or 40 hex digit keyid to find a key for 1206 | 1207 | Returns: 1208 | A string containing key material for the specified GPG key id 1209 | 1210 | 1211 | Raises: 1212 | subprocess.CalledProcessError 1213 | """ 1214 | # options=mr - machine-readable output (disables html wrappers) 1215 | keyserver_url = ( 1216 | "https://keyserver.ubuntu.com" "/pks/lookup?op=get&options=mr&exact=on&search=0x{}" 1217 | ) 1218 | curl_cmd = ["curl", keyserver_url.format(keyid)] 1219 | with tracer.start_as_current_span(curl_cmd[0]) as span: 1220 | span.set_attribute("argv", curl_cmd) 1221 | # use proxy server settings in order to retrieve the key 1222 | return check_output(curl_cmd).decode() 1223 | 1224 | @staticmethod 1225 | def _dearmor_gpg_key(key_asc: bytes) -> bytes: 1226 | """Convert a GPG key in the ASCII armor format to the binary format. 1227 | 1228 | Args: 1229 | key_asc: A GPG key in ASCII armor format. 1230 | 1231 | Returns: 1232 | A GPG key in binary format as a string 1233 | 1234 | Raises: 1235 | GPGKeyError 1236 | """ 1237 | cmd = ["gpg", "--dearmor"] 1238 | with tracer.start_as_current_span(cmd[0]) as span: 1239 | span.set_attribute("argv", cmd) 1240 | ps = subprocess.run(cmd, capture_output=True, input=key_asc) 1241 | out, err = ps.stdout, ps.stderr.decode() 1242 | if "gpg: no valid OpenPGP data found." in err: 1243 | raise GPGKeyError( 1244 | "Invalid GPG key material. Check your network setup" 1245 | " (MTU, routing, DNS) and/or proxy server settings" 1246 | " as well as destination keyserver status." 1247 | ) 1248 | else: 1249 | return out 1250 | 1251 | @staticmethod 1252 | def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: 1253 | """Write GPG key material into a file at a provided path. 1254 | 1255 | Args: 1256 | key_name: A key name to use for a key file (could be a fingerprint) 1257 | key_material: A GPG key material (binary) 1258 | """ 1259 | with open(key_name, "wb") as keyf: 1260 | keyf.write(key_material) 1261 | 1262 | 1263 | def _repo_to_identifier(repo: DebianRepository) -> str: 1264 | """Return str identifier derived from repotype, uri, and release. 1265 | 1266 | Private method used to produce the identifiers used by RepositoryMapping. 1267 | """ 1268 | return f"{repo.repotype}-{repo.uri}-{repo.release}" 1269 | 1270 | 1271 | def _repo_to_line(repo: DebianRepository, include_signed_by: bool = True) -> str: 1272 | """Return the one-per-line format repository definition.""" 1273 | return "{prefix}{repotype} {options}{uri} {release} {groups}".format( 1274 | prefix="" if repo.enabled else "#", 1275 | repotype=repo.repotype, 1276 | options=repo.make_options_string(include_signed_by=include_signed_by), 1277 | uri=repo.uri, 1278 | release=repo.release, 1279 | groups=" ".join(repo.groups), 1280 | ) 1281 | 1282 | 1283 | class RepositoryMapping(Mapping[str, DebianRepository]): 1284 | """An representation of known repositories. 1285 | 1286 | Instantiation of `RepositoryMapping` will iterate through the 1287 | filesystem, parse out repository files in `/etc/apt/...`, and create 1288 | `DebianRepository` objects in this list. 1289 | 1290 | Typical usage: 1291 | 1292 | repositories = apt.RepositoryMapping() 1293 | repositories.add(DebianRepository( 1294 | enabled=True, repotype="deb", uri="https://example.com", release="focal", 1295 | groups=["universe"] 1296 | )) 1297 | """ 1298 | 1299 | _apt_dir = "/etc/apt" 1300 | _sources_subdir = "sources.list.d" 1301 | _default_list_name = "sources.list" 1302 | _default_sources_name = "ubuntu.sources" 1303 | _last_errors: tuple[Error, ...] = () 1304 | 1305 | def __init__(self): 1306 | self._repository_map: dict[str, DebianRepository] = {} 1307 | self.default_file = os.path.join(self._apt_dir, self._default_list_name) 1308 | # ^ public attribute for backwards compatibility only 1309 | sources_dir = os.path.join(self._apt_dir, self._sources_subdir) 1310 | default_sources = os.path.join(sources_dir, self._default_sources_name) 1311 | 1312 | # read sources.list if it exists 1313 | # ignore InvalidSourceError if ubuntu.sources also exists 1314 | # -- in this case, sources.list just contains a comment 1315 | if os.path.isfile(self.default_file): 1316 | try: 1317 | self.load(self.default_file) 1318 | except InvalidSourceError: 1319 | if not os.path.isfile(default_sources): 1320 | raise 1321 | 1322 | with tracer.start_as_current_span("load sources"): 1323 | # read sources.list.d 1324 | for file in glob.iglob(os.path.join(sources_dir, "*.list")): 1325 | self.load(file) 1326 | for file in glob.iglob(os.path.join(sources_dir, "*.sources")): 1327 | self.load_deb822(file) 1328 | 1329 | def __contains__(self, key: Any) -> bool: 1330 | """Magic method for checking presence of repo in mapping. 1331 | 1332 | Checks against the string names used to identify repositories. 1333 | """ 1334 | return key in self._repository_map 1335 | 1336 | def __len__(self) -> int: 1337 | """Return number of repositories in map.""" 1338 | return len(self._repository_map) 1339 | 1340 | def __iter__(self) -> Iterator[DebianRepository]: # pyright: ignore[reportIncompatibleMethodOverride] 1341 | """Return iterator for RepositoryMapping. 1342 | 1343 | Iterates over the DebianRepository values rather than the string names. 1344 | FIXME: this breaks the expectations of the Mapping abstract base class 1345 | for example when it provides methods like keys and items 1346 | """ 1347 | return iter(self._repository_map.values()) 1348 | 1349 | def __getitem__(self, repository_uri: str) -> DebianRepository: 1350 | """Return a given `DebianRepository`.""" 1351 | return self._repository_map[repository_uri] 1352 | 1353 | def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None: 1354 | """Add a `DebianRepository` to the cache.""" 1355 | self._repository_map[repository_uri] = repository 1356 | 1357 | def load_deb822(self, filename: str) -> None: 1358 | """Load a deb822 format repository source file into the cache. 1359 | 1360 | In contrast to one-line-style, the deb822 format specifies a repository 1361 | using a multi-line stanza. Stanzas are separated by whitespace, 1362 | and each definition consists of lines that are either key: value pairs, 1363 | or continuations of the previous value. 1364 | 1365 | Read more about the deb822 format here: 1366 | https://manpages.ubuntu.com/manpages/noble/en/man5/sources.list.5.html 1367 | For instance, ubuntu 24.04 (noble) lists its sources using deb822 style in: 1368 | /etc/apt/sources.list.d/ubuntu.sources 1369 | """ 1370 | with open(filename) as f: 1371 | repos, errors = self._parse_deb822_lines(f, filename=filename) 1372 | for repo in repos: 1373 | self._repository_map[_repo_to_identifier(repo)] = repo 1374 | if errors: 1375 | self._last_errors = tuple(errors) 1376 | logger.debug( 1377 | "the following %d error(s) were encountered when reading deb822 sources:\n%s", 1378 | len(errors), 1379 | "\n".join(str(e) for e in errors), 1380 | ) 1381 | if repos: 1382 | logger.info("parsed %d apt package repositories from %s", len(repos), filename) 1383 | else: 1384 | raise InvalidSourceError(f"all repository lines in '{filename}' were invalid!") 1385 | 1386 | @classmethod 1387 | def _parse_deb822_lines( 1388 | cls, 1389 | lines: Iterable[str], 1390 | filename: str = "", 1391 | ) -> tuple[list[DebianRepository], list[InvalidSourceError]]: 1392 | """Parse lines from a deb822 file into a list of repos and a list of errors. 1393 | 1394 | The semantics of `_parse_deb822_lines` slightly different to `_parse`: 1395 | `_parse` reads a commented out line as an entry that is not enabled 1396 | `_parse_deb822_lines` strips out comments entirely when parsing a file into stanzas, 1397 | instead only reading the 'Enabled' key to determine if an entry is enabled 1398 | """ 1399 | repos: list[DebianRepository] = [] 1400 | errors: list[InvalidSourceError] = [] 1401 | for numbered_lines in _iter_deb822_stanzas(lines): 1402 | try: 1403 | stanza = _Deb822Stanza(numbered_lines=numbered_lines, filename=filename) 1404 | except InvalidSourceError as e: # noqa: PERF203 1405 | errors.append(e) 1406 | else: 1407 | repos.extend(stanza.repos) 1408 | return repos, errors 1409 | 1410 | def load(self, filename: str): 1411 | """Load a one-line-style format repository source file into the cache. 1412 | 1413 | Args: 1414 | filename: the path to the repository file 1415 | """ 1416 | parsed: list[int] = [] 1417 | skipped: list[int] = [] 1418 | with open(filename) as f: 1419 | for n, line in enumerate(f, start=1): # 1 indexed line numbers 1420 | try: 1421 | repo = self._parse(line, filename) 1422 | except InvalidSourceError: # noqa: PERF203 1423 | skipped.append(n) 1424 | else: 1425 | repo_identifier = _repo_to_identifier(repo) 1426 | self._repository_map[repo_identifier] = repo 1427 | parsed.append(n) 1428 | logger.debug("parsed repo: '%s'", repo_identifier) 1429 | 1430 | if skipped: 1431 | skip_list = ", ".join(str(s) for s in skipped) 1432 | logger.debug("skipped the following lines in file '%s': %s", filename, skip_list) 1433 | 1434 | if parsed: 1435 | logger.info("parsed %d apt package repositories from %s", len(parsed), filename) 1436 | else: 1437 | raise InvalidSourceError(f"all repository lines in '{filename}' were invalid!") 1438 | 1439 | @staticmethod 1440 | def _parse(line: str, filename: str) -> DebianRepository: 1441 | """Parse a line in a sources.list file. 1442 | 1443 | Args: 1444 | line: a single line from `load` to parse 1445 | filename: the filename being read 1446 | 1447 | Raises: 1448 | InvalidSourceError if the source type is unknown 1449 | """ 1450 | enabled = True 1451 | repotype = uri = release = gpg_key = "" 1452 | options = {} 1453 | groups = [] 1454 | 1455 | line = line.strip() 1456 | if line.startswith("#"): 1457 | enabled = False 1458 | line = line[1:] 1459 | 1460 | # Check for "#" in the line and treat a part after it as a comment then strip it off. 1461 | i = line.find("#") 1462 | if i > 0: 1463 | line = line[:i] 1464 | 1465 | # Split a source into substrings to initialize a new repo. 1466 | source = line.strip() 1467 | if source: 1468 | # Match any repo options, and get a dict representation. 1469 | for v in re.findall(OPTIONS_MATCHER, source): 1470 | opts = dict(o.split("=") for o in v.strip("[]").split()) 1471 | # Extract the 'signed-by' option for the gpg_key 1472 | gpg_key = opts.pop("signed-by", "") 1473 | options = opts 1474 | 1475 | # Remove any options from the source string and split the string into chunks 1476 | source = re.sub(OPTIONS_MATCHER, "", source) 1477 | chunks = source.split() 1478 | 1479 | # Check we've got a valid list of chunks 1480 | if len(chunks) < 3 or chunks[0] not in VALID_SOURCE_TYPES: 1481 | raise InvalidSourceError("An invalid sources line was found in %s!", filename) 1482 | 1483 | repotype = chunks[0] 1484 | uri = chunks[1] 1485 | release = chunks[2] 1486 | groups = chunks[3:] 1487 | 1488 | return DebianRepository( 1489 | enabled, repotype, uri, release, groups, filename, gpg_key, options 1490 | ) 1491 | else: 1492 | raise InvalidSourceError("An invalid sources line was found in %s!", filename) 1493 | 1494 | def add( # noqa: D417 # undocumented-param: default_filename intentionally undocumented 1495 | self, repo: DebianRepository, default_filename: bool | None = False 1496 | ) -> None: 1497 | """Add a new repository to the system using add-apt-repository. 1498 | 1499 | Args: 1500 | repo: a DebianRepository object 1501 | if repo.enabled is falsey, will return without adding the repository 1502 | Raises: 1503 | CalledProcessError: if there's an error running apt-add-repository 1504 | 1505 | WARNING: Does not associate the repository with a signing key. 1506 | Use `import_key` to add a signing key globally. 1507 | 1508 | WARNING: if repo.enabled is falsey, will return without adding the repository 1509 | 1510 | WARNING: Don't forget to call `apt.update` before installing any packages! 1511 | Or call `apt.add_package` with `update_cache=True`. 1512 | 1513 | WARNING: the default_filename keyword argument is provided for backwards compatibility 1514 | only. It is not used, and was not used in the previous revision of this library. 1515 | """ 1516 | if not repo.enabled: 1517 | logger.warning( 1518 | ( 1519 | "Returning from RepositoryMapping.add(repo=%s) without adding the repo" 1520 | " because repo.enabled is %s" 1521 | ), 1522 | repo, 1523 | repo.enabled, 1524 | ) 1525 | return 1526 | _add_repository(repo) 1527 | self._repository_map[_repo_to_identifier(repo)] = repo 1528 | 1529 | def disable(self, repo: DebianRepository) -> None: 1530 | """Remove a repository by disabling it in the source file. 1531 | 1532 | WARNING: disable is currently not implemented for repositories defined 1533 | by a deb822 stanza, and will raise a NotImplementedError if called on one. 1534 | 1535 | WARNING: This method does NOT alter the `.enabled` flag on the DebianRepository. 1536 | """ 1537 | repo.disable() 1538 | self._repository_map[_repo_to_identifier(repo)] = repo 1539 | # ^ adding to map on disable seems like a bug, but this is the previous behaviour 1540 | 1541 | 1542 | def _add_repository( 1543 | repo: DebianRepository, 1544 | remove: bool = False, 1545 | update_cache: bool = False, 1546 | ) -> None: 1547 | line = _repo_to_line(repo, include_signed_by=False) 1548 | key_file = repo.gpg_key 1549 | if key_file and not remove and not os.path.exists(key_file): 1550 | msg = ( 1551 | "Adding repository '%s' with add-apt-repository." 1552 | " Key file '%s' does not exist." 1553 | " Ensure it is imported correctly to use this repository." 1554 | ) 1555 | logger.warning(msg, line, key_file) 1556 | cmd = [ 1557 | "add-apt-repository", 1558 | "--yes", 1559 | "--sourceslist=" + line, 1560 | ] 1561 | if remove: 1562 | cmd.append("--remove") 1563 | if not update_cache: 1564 | cmd.append("--no-update") 1565 | logger.info("%s", cmd) 1566 | try: 1567 | with tracer.start_as_current_span(cmd[0]) as span: 1568 | span.set_attribute("argv", cmd) 1569 | subprocess.run(cmd, check=True, capture_output=True) 1570 | except CalledProcessError as e: 1571 | logger.error( 1572 | "subprocess.run(%s):\nstdout:\n%s\nstderr:\n%s", 1573 | cmd, 1574 | e.stdout.decode(), 1575 | e.stderr.decode(), 1576 | ) 1577 | raise 1578 | 1579 | 1580 | class _Deb822Stanza: 1581 | """Representation of a stanza from a deb822 source file. 1582 | 1583 | May define multiple DebianRepository objects. 1584 | """ 1585 | 1586 | def __init__(self, numbered_lines: list[tuple[int, str]], filename: str = ""): 1587 | self._filename = filename 1588 | self._numbered_lines = numbered_lines 1589 | if not numbered_lines: 1590 | self._repos = () 1591 | self._gpg_key_filename = "" 1592 | self._gpg_key_from_stanza = None 1593 | return 1594 | options, line_numbers = _deb822_stanza_to_options(numbered_lines) 1595 | repos, gpg_key_info = _deb822_options_to_repos( 1596 | options, line_numbers=line_numbers, filename=filename 1597 | ) 1598 | for repo in repos: 1599 | repo._deb822_stanza = self 1600 | self._repos = repos 1601 | self._gpg_key_filename, self._gpg_key_from_stanza = gpg_key_info 1602 | 1603 | @property 1604 | def repos(self) -> tuple[DebianRepository, ...]: 1605 | """The repositories defined by this deb822 stanza.""" 1606 | return self._repos 1607 | 1608 | def get_gpg_key_filename(self) -> str: 1609 | """Return the path to the GPG key for this stanza. 1610 | 1611 | Import the key first, if the key itself was provided in the stanza. 1612 | Return an empty string if no filename or key was provided. 1613 | """ 1614 | if self._gpg_key_filename: 1615 | return self._gpg_key_filename 1616 | if self._gpg_key_from_stanza is None: 1617 | return "" 1618 | # a gpg key was provided in the stanza 1619 | # and we haven't already imported it 1620 | self._gpg_key_filename = import_key(self._gpg_key_from_stanza) 1621 | return self._gpg_key_filename 1622 | 1623 | 1624 | class MissingRequiredKeyError(InvalidSourceError): 1625 | """Missing a required value in a source file.""" 1626 | 1627 | def __init__(self, message: str = "", *, file: str, line: int | None, key: str) -> None: 1628 | super().__init__(message, file, line, key) 1629 | self.file = file 1630 | self.line = line 1631 | self.key = key 1632 | 1633 | 1634 | class BadValueError(InvalidSourceError): 1635 | """Bad value for an entry in a source file.""" 1636 | 1637 | def __init__( 1638 | self, 1639 | message: str = "", 1640 | *, 1641 | file: str, 1642 | line: int | None, 1643 | key: str, 1644 | value: str, 1645 | ) -> None: 1646 | super().__init__(message, file, line, key, value) 1647 | self.file = file 1648 | self.line = line 1649 | self.key = key 1650 | self.value = value 1651 | 1652 | 1653 | def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[list[tuple[int, str]]]: 1654 | """Given lines from a deb822 format file, yield a stanza of lines. 1655 | 1656 | Args: 1657 | lines: an iterable of lines from a deb822 sources file 1658 | 1659 | Yields: 1660 | lists of numbered lines (a tuple of line number and line) that make up 1661 | a deb822 stanza, with comments stripped out (but accounted for in line numbering) 1662 | """ 1663 | current_stanza: list[tuple[int, str]] = [] 1664 | for n, line in enumerate(lines, start=1): # 1 indexed line numbers 1665 | if not line.strip(): # blank lines separate stanzas 1666 | if current_stanza: 1667 | yield current_stanza 1668 | current_stanza = [] 1669 | continue 1670 | content, _delim, _comment = line.partition("#") 1671 | if content.strip(): # skip (potentially indented) comment line 1672 | current_stanza.append((n, content.rstrip())) # preserve indent 1673 | if current_stanza: 1674 | yield current_stanza 1675 | 1676 | 1677 | def _deb822_stanza_to_options( 1678 | lines: Iterable[tuple[int, str]], 1679 | ) -> tuple[dict[str, str], dict[str, int]]: 1680 | """Turn numbered lines into a dict of options and a dict of line numbers. 1681 | 1682 | Args: 1683 | lines: an iterable of numbered lines (a tuple of line number and line) 1684 | 1685 | Returns: 1686 | a dictionary of option names to (potentially multiline) values, and 1687 | a dictionary of option names to starting line number 1688 | """ 1689 | parts: dict[str, list[str]] = {} 1690 | line_numbers: dict[str, int] = {} 1691 | current = None 1692 | for n, line in lines: 1693 | assert "#" not in line # comments should be stripped out 1694 | if line.startswith(" "): # continuation of previous key's value 1695 | assert current is not None 1696 | parts[current].append(line.rstrip()) # preserve indent 1697 | continue 1698 | raw_key, _, raw_value = line.partition(":") 1699 | current = raw_key.strip() 1700 | parts[current] = [raw_value.strip()] 1701 | line_numbers[current] = n 1702 | options = {k: "\n".join(v) for k, v in parts.items()} 1703 | return options, line_numbers 1704 | 1705 | 1706 | def _deb822_options_to_repos( 1707 | options: dict[str, str], line_numbers: Mapping[str, int] = {}, filename: str = "" 1708 | ) -> tuple[tuple[DebianRepository, ...], tuple[str, str | None]]: 1709 | """Return a collections of DebianRepository objects defined by this deb822 stanza. 1710 | 1711 | Args: 1712 | options: a dictionary of deb822 field names to string options 1713 | line_numbers: a dictionary of field names to line numbers (for error messages) 1714 | filename: the file the options were read from (for repository object and errors) 1715 | 1716 | Returns: 1717 | a tuple of `DebianRepository`s, and 1718 | a tuple of the gpg key filename and optional in-stanza provided key itself 1719 | 1720 | Raises: 1721 | InvalidSourceError if any options are malformed or required options are missing 1722 | """ 1723 | # Enabled 1724 | enabled_field = options.pop("Enabled", "yes") 1725 | if enabled_field == "yes": 1726 | enabled = True 1727 | elif enabled_field == "no": 1728 | enabled = False 1729 | else: 1730 | raise BadValueError( 1731 | "Must be one of yes or no (default: yes).", 1732 | file=filename, 1733 | line=line_numbers.get("Enabled"), 1734 | key="Enabled", 1735 | value=enabled_field, 1736 | ) 1737 | # Signed-By 1738 | gpg_key_file = options.pop("Signed-By", "") 1739 | gpg_key_from_stanza: str | None = None 1740 | if "\n" in gpg_key_file: 1741 | # actually a literal multi-line gpg-key rather than a filename 1742 | gpg_key_from_stanza = gpg_key_file 1743 | gpg_key_file = "" 1744 | # Types 1745 | try: 1746 | repotypes = options.pop("Types").split() 1747 | uris = options.pop("URIs").split() 1748 | suites = options.pop("Suites").split() 1749 | except KeyError as e: 1750 | [key] = e.args 1751 | raise MissingRequiredKeyError( 1752 | key=key, 1753 | line=min(line_numbers.values()) if line_numbers else None, 1754 | file=filename, 1755 | ) from e 1756 | # Components 1757 | # suite can specify an exact path, in which case the components must be omitted 1758 | # and suite must end with a slash (/). 1759 | # If suite does not specify an exact path, at least one component must be present. 1760 | # https://manpages.ubuntu.com/manpages/noble/man5/sources.list.5.html 1761 | components: list[str] 1762 | if len(suites) == 1 and suites[0].endswith("/"): 1763 | if "Components" in options: 1764 | msg = ( 1765 | "Since 'Suites' (line {suites_line}) specifies" 1766 | " a path relative to 'URIs' (line {uris_line})," 1767 | " 'Components' must be omitted." 1768 | ).format( 1769 | suites_line=line_numbers.get("Suites"), 1770 | uris_line=line_numbers.get("URIs"), 1771 | ) 1772 | raise BadValueError( 1773 | msg, 1774 | file=filename, 1775 | line=line_numbers.get("Components"), 1776 | key="Components", 1777 | value=options["Components"], 1778 | ) 1779 | components = [] 1780 | else: 1781 | if "Components" not in options: 1782 | msg = ( 1783 | "Since 'Suites' (line {suites_line}) does not specify" 1784 | " a path relative to 'URIs' (line {uris_line})," 1785 | " 'Components' must be present in this stanza." 1786 | ).format( 1787 | suites_line=line_numbers.get("Suites"), 1788 | uris_line=line_numbers.get("URIs"), 1789 | ) 1790 | raise MissingRequiredKeyError( 1791 | msg, 1792 | file=filename, 1793 | line=min(line_numbers.values()) if line_numbers else None, 1794 | key="Components", 1795 | ) 1796 | components = options.pop("Components").split() 1797 | repos = tuple( 1798 | DebianRepository( 1799 | enabled=enabled, 1800 | repotype=repotype, 1801 | uri=uri, 1802 | release=suite, 1803 | groups=components, 1804 | filename=filename, 1805 | gpg_key_filename=gpg_key_file, 1806 | options=options, 1807 | ) 1808 | for repotype in repotypes 1809 | for uri in uris 1810 | for suite in suites 1811 | ) 1812 | return repos, (gpg_key_file, gpg_key_from_stanza) 1813 | --------------------------------------------------------------------------------