├── README.md ├── requirements.txt ├── .gitignore ├── scripts ├── deploy.sh └── refresh.sh ├── src ├── utils.py ├── entities.py ├── grafana_dashboards │ └── Monitoring.json.tmpl ├── charm.py └── workload.py ├── .github └── workflows │ ├── release.yaml │ └── pull-request.yaml ├── pyproject.toml ├── charmcraft.yaml ├── tests ├── unit │ ├── test_ratings.py │ └── test_charm.py └── integration │ └── test_charm.py ├── tox.ini ├── LICENSE └── lib └── charms ├── traefik_route_k8s └── v0 │ └── traefik_route.py └── grafana_k8s └── v0 └── grafana_dashboard.py /README.md: -------------------------------------------------------------------------------- 1 | # K8s operator for the Ubuntu App Ratings service 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ops ==2.14.1 2 | psycopg[binary] ==3.1.19 3 | requests ~= 2.31 4 | types-requests ~= 2.31 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .idea 3 | /target 4 | .env 5 | .env_test 6 | .DS_STORE 7 | 8 | *.rock 9 | *.nix 10 | 11 | venv/ 12 | build/ 13 | *.charm 14 | .tox/ 15 | .coverage 16 | __pycache__/ 17 | *.py[cod] 18 | .mypy_cache 19 | -------------------------------------------------------------------------------- /scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | TAG="sha-${1}" 6 | 7 | charmcraft pack 8 | 9 | juju deploy \ 10 | ./ubuntu-app-center-ratings_ubuntu-22.04-amd64.charm \ 11 | app-ratings \ 12 | --model=desktop \ 13 | --resource "image=ghcr.io/tim-hm/app-center-ratings:$TAG" 14 | -------------------------------------------------------------------------------- /scripts/refresh.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -euxo pipefail 4 | 5 | TAG="sha-${1}" 6 | 7 | charmcraft pack 8 | 9 | juju refresh \ 10 | app-ratings \ 11 | --model=desktop \ 12 | --path=./ubuntu-app-ratings_ubuntu-22.04-amd64.charm \ 13 | --force-units \ 14 | --resource "image=ghcr.io/tim-hm/app-center-ratings:$TAG" 15 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, TypeVar 2 | 3 | T = TypeVar("T") 4 | 5 | 6 | def get_or_fail(value: Optional[T], name: str = "unknown") -> T: 7 | if value is None: 8 | raise ValueError(f"{name} was None") 9 | return value 10 | 11 | 12 | def get_or_default(value: Optional[T], default: T) -> T: 13 | if value is None: 14 | return default 15 | return value 16 | 17 | 18 | def stringify(instance: Any) -> str: 19 | class_name = instance.__class__.__name__ 20 | properties_str = ", ".join(f"{attr}={value}" for attr, value in instance.__dict__.items()) 21 | return f"{class_name}{{{properties_str}}}" 22 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | name: Release to latest/candidate 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v2 15 | 16 | - uses: canonical/charming-actions/release-libraries@2.5.0-rc 17 | with: 18 | credentials: "${{ secrets.CHARMHUB_TOKEN }}" 19 | github-token: "${{ secrets.GITHUB_TOKEN }}" 20 | 21 | - uses: canonical/charming-actions/upload-charm@2.5.0-rc 22 | with: 23 | credentials: "${{ secrets.CHARMHUB_TOKEN }}" 24 | github-token: "${{ secrets.GITHUB_TOKEN }}" 25 | channel: "latest/candidate" 26 | -------------------------------------------------------------------------------- /.github/workflows/pull-request.yaml: -------------------------------------------------------------------------------- 1 | name: Pull Request 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | name: Release to latest/candidate 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | 15 | - uses: canonical/charming-actions/release-libraries@2.5.0-rc 16 | with: 17 | credentials: "${{ secrets.CHARMHUB_TOKEN }}" 18 | github-token: "${{ secrets.GITHUB_TOKEN }}" 19 | 20 | - uses: canonical/charming-actions/upload-charm@2.5.0-rc 21 | with: 22 | credentials: "${{ secrets.CHARMHUB_TOKEN }}" 23 | github-token: "${{ secrets.GITHUB_TOKEN }}" 24 | upload-image: "true" 25 | channel: "${{ steps.channel.outputs.name }}" 26 | -------------------------------------------------------------------------------- /src/entities.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import Optional 4 | 5 | 6 | @dataclass 7 | class About: 8 | version: str 9 | 10 | 11 | class WorkloadEnv(Enum): 12 | Prod = "prod" 13 | Stg = "stg" 14 | Local = "local" 15 | 16 | @classmethod 17 | def try_from_string(cls, value: str) -> Optional["WorkloadEnv"]: 18 | value = value.lower() 19 | 20 | for member in cls: 21 | if member.value == value: 22 | return member 23 | 24 | return None 25 | 26 | 27 | class LogLevel(Enum): 28 | Debug = "debug" 29 | Info = "info" 30 | 31 | @classmethod 32 | def try_from_string(cls, value: str) -> "LogLevel": 33 | value = value.lower() 34 | 35 | for member in cls: 36 | if member.value == value: 37 | return member 38 | 39 | return LogLevel.Info 40 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Testing tools configuration 2 | [tool.coverage.run] 3 | branch = true 4 | 5 | [tool.coverage.report] 6 | show_missing = true 7 | 8 | [tool.pytest.ini_options] 9 | minversion = "6.0" 10 | log_cli_level = "INFO" 11 | 12 | # Formatting tools configuration 13 | [tool.black] 14 | line-length = 99 15 | target-version = ["py38"] 16 | 17 | # Linting tools configuration 18 | [tool.ruff] 19 | line-length = 99 20 | select = ["E", "W", "F", "C", "N", "D", "I001"] 21 | extend-ignore = [ 22 | "D100", 23 | "D101", 24 | "D102", 25 | "D103", 26 | "D203", 27 | "D204", 28 | "D213", 29 | "D215", 30 | "D400", 31 | "D404", 32 | "D406", 33 | "D407", 34 | "D408", 35 | "D409", 36 | "D413", 37 | ] 38 | ignore = ["E501", "D107"] 39 | extend-exclude = ["__pycache__", "*.egg_info"] 40 | per-file-ignores = { "tests/*" = ["D100", "D101", "D102", "D103", "D104"] } 41 | 42 | [tool.ruff.mccabe] 43 | max-complexity = 10 44 | 45 | [tool.codespell] 46 | skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.coverage" 47 | 48 | [tool.pyright] 49 | include = ["src/**.py"] 50 | -------------------------------------------------------------------------------- /charmcraft.yaml: -------------------------------------------------------------------------------- 1 | title: Ubuntu App Center Ratings Service 2 | name: ubuntu-app-ratings 3 | type: charm 4 | summary: A ratings service for snapped applications 5 | description: | 6 | This k8s operator charm wraps the app ratings service. 7 | 8 | The app ratings service is the backend for the App Center's snap ratings feature. 9 | 10 | assumes: 11 | - juju >= 3.1 12 | 13 | bases: 14 | - build-on: 15 | - name: ubuntu 16 | channel: "22.04" 17 | run-on: 18 | - name: ubuntu 19 | channel: "22.04" 20 | 21 | config: 22 | options: 23 | env: 24 | description: The charm's environment. One of prod, stg, or local 25 | type: string 26 | log_level: 27 | default: info 28 | description: Set the workload's logging verbosity. One of debug or info. 29 | type: string 30 | jwt_secret: 31 | description: Base64 JWT secret 32 | type: string 33 | 34 | containers: 35 | workload: 36 | resource: image 37 | 38 | resources: 39 | image: 40 | type: oci-image 41 | description: OCI image for the ratings service 42 | upstream-source: ghcr.io/tim-hm/app-center-ratings:sha-0f696eb 43 | 44 | requires: 45 | database: 46 | interface: postgresql_client 47 | limit: 1 48 | log-proxy: 49 | interface: loki_push_api 50 | limit: 1 51 | ingress: 52 | interface: traefik_route 53 | limit: 1 54 | 55 | provides: 56 | metrics-endpoint: 57 | interface: prometheus_scrape 58 | grafana-dashboard: 59 | interface: grafana_dashboard 60 | -------------------------------------------------------------------------------- /tests/unit/test_ratings.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Canonical 2 | # See LICENSE file for licensing details. 3 | 4 | import unittest 5 | 6 | # from unittest.mock import patch 7 | from ratings import Ratings 8 | 9 | 10 | class TestRatings(unittest.TestCase): 11 | def setUp(self): 12 | pass 13 | 14 | def test_ratings_constructor_props(self): 15 | r = Ratings("foobar", "deadbeef") 16 | self.assertEqual(r.connection_string, "foobar") 17 | self.assertEqual(r.jwt_secret, "deadbeef") 18 | 19 | def test_ratings_ready_connection_string_present_jwt_not_present(self): 20 | r = Ratings("foobar", "") 21 | self.assertFalse(r.ready()) 22 | 23 | def test_ratings_ready_connection_string_not_present_jwt_present(self): 24 | r = Ratings("", "foobar") 25 | self.assertFalse(r.ready()) 26 | 27 | def test_ratings_ready_connection_string_present_jwt(self): 28 | r = Ratings("foobar", "deadbeef") 29 | self.assertTrue(r.ready()) 30 | 31 | def test_ratings_pebble_layer(self): 32 | r = Ratings("foobar", "deadbeef") 33 | self.assertEqual( 34 | r.pebble_layer(), 35 | { 36 | "summary": "ratings layer", 37 | "description": "pebble config layer for ratings", 38 | "services": { 39 | "ratings": { 40 | "override": "replace", 41 | "summary": "ratings", 42 | "command": "/bin/ratings", 43 | "startup": "enabled", 44 | "environment": { 45 | "APP_ENV": "dev", 46 | "APP_JWT_SECRET": "deadbeef", 47 | "APP_POSTGRES_URI": "foobar", 48 | "APP_MIGRATION_POSTGRES_URI": "foobar", 49 | "APP_LOG_LEVEL": "info", 50 | }, 51 | } 52 | }, 53 | }, 54 | ) 55 | -------------------------------------------------------------------------------- /src/grafana_dashboards/Monitoring.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": null, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "prometheus", 34 | "uid": "${prometheusds}" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "custom": {}, 39 | "mappings": [], 40 | "thresholds": { 41 | "steps": [ 42 | { 43 | "color": "green", 44 | "value": null 45 | } 46 | ] 47 | } 48 | }, 49 | "overrides": [] 50 | }, 51 | "gridPos": { 52 | "h": 8, 53 | "w": 24, 54 | "x": 0, 55 | "y": 0 56 | }, 57 | "id": 1, 58 | "options": { 59 | "displayMode": "lcd", 60 | "reduceOptions": { 61 | "calcs": [ 62 | "lastNotNull" 63 | ], 64 | "fields": "", 65 | "values": false 66 | }, 67 | "showUnfilled": true 68 | }, 69 | "targets": [ 70 | { 71 | "expr": "app_ratings_requests_total", 72 | "interval": "", 73 | "legendFormat": "Total Requests", 74 | "refId": "A" 75 | }, 76 | { 77 | "expr": "app_ratings_errors_total", 78 | "interval": "", 79 | "legendFormat": "Total Errors", 80 | "refId": "B" 81 | }, 82 | { 83 | "expr": "app_ratings_uptime_seconds", 84 | "interval": "", 85 | "legendFormat": "Uptime (seconds)", 86 | "refId": "C" 87 | } 88 | ], 89 | "title": "App Ratings", 90 | "type": "stat" 91 | } 92 | ], 93 | "refresh": "10s", 94 | "schemaVersion": 37, 95 | "style": "dark", 96 | "tags": [], 97 | "templating": { 98 | "list": [] 99 | }, 100 | "time": { 101 | "from": "now-1h", 102 | "to": "now" 103 | }, 104 | "timepicker": {}, 105 | "timezone": "", 106 | "title": "App Center Monitoring", 107 | "uid": "LPGoVNc4z", 108 | "version": 10, 109 | "weekStart": "" 110 | } 111 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Canonical 2 | # See LICENSE file for licensing details. 3 | 4 | [tox] 5 | no_package = True 6 | skip_missing_interpreters = True 7 | env_list = format, lint, unit 8 | min_version = 4.0.0 9 | 10 | [vars] 11 | src_path = {tox_root}/src 12 | tests_path = {tox_root}/tests 13 | ;lib_path = {tox_root}/lib/charms/operator_name_with_underscores 14 | all_path = {[vars]src_path} {[vars]tests_path} 15 | 16 | [testenv] 17 | set_env = 18 | PYTHONPATH = {tox_root}/lib:{[vars]src_path} 19 | PYTHONBREAKPOINT=pdb.set_trace 20 | PY_COLORS=1 21 | pass_env = 22 | PYTHONPATH 23 | CHARM_BUILD_DIR 24 | MODEL_SETTINGS 25 | GITHUB_ACTION 26 | 27 | [testenv:format] 28 | description = Apply coding style standards to code 29 | deps = 30 | black 31 | ruff 32 | commands = 33 | black {[vars]all_path} 34 | ruff --fix {[vars]all_path} 35 | 36 | [testenv:lint] 37 | description = Check code against coding style standards 38 | deps = 39 | black 40 | ruff 41 | codespell 42 | commands = 43 | # if this charm owns a lib, uncomment "lib_path" variable 44 | # and uncomment the following line 45 | # codespell {[vars]lib_path} 46 | codespell {tox_root} 47 | ruff {[vars]all_path} 48 | black --check --diff {[vars]all_path} 49 | 50 | [testenv:unit] 51 | description = Run unit tests 52 | deps = 53 | pytest 54 | coverage[toml] 55 | pydantic 56 | -r {tox_root}/requirements.txt 57 | commands = 58 | coverage run --source={[vars]src_path} \ 59 | -m pytest \ 60 | --tb native \ 61 | -v \ 62 | -s \ 63 | {posargs} \ 64 | {[vars]tests_path}/unit 65 | coverage report 66 | 67 | [testenv:integration] 68 | description = Run integration tests 69 | deps = 70 | pytest 71 | juju 72 | pytest-operator 73 | grpcio 74 | -r {tox_root}/requirements.txt 75 | commands = 76 | pytest -v \ 77 | -s \ 78 | --tb native \ 79 | --log-cli-level=INFO \ 80 | {posargs} \ 81 | {[vars]tests_path}/integration 82 | 83 | [testenv:grpc] 84 | description = Regenerate gRPC stubs 85 | deps = 86 | grpcio 87 | grpcio-tools 88 | commands = 89 | python3 -m grpc_tools.protoc \ 90 | --proto_path={tox_root}/../proto \ 91 | --python_out={tox_root}/lib/ratings_api \ 92 | --grpc_python_out={tox_root}/lib/ratings_api \ 93 | {tox_root}/../proto/ratings_features_app.proto 94 | 95 | python3 -m grpc_tools.protoc \ 96 | --proto_path={tox_root}/../proto \ 97 | --python_out={tox_root}/lib/ratings_api \ 98 | --grpc_python_out={tox_root}/lib/ratings_api \ 99 | {tox_root}/../proto/ratings_features_chart.proto 100 | 101 | python3 -m grpc_tools.protoc \ 102 | --proto_path={tox_root}/../proto \ 103 | --python_out={tox_root}/lib/ratings_api \ 104 | --grpc_python_out={tox_root}/lib/ratings_api \ 105 | {tox_root}/../proto/ratings_features_user.proto 106 | -------------------------------------------------------------------------------- /tests/integration/test_charm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2023 Canonical 3 | # See LICENSE file for licensing details. 4 | 5 | import asyncio 6 | import json 7 | import logging 8 | import os 9 | import secrets 10 | from pathlib import Path 11 | 12 | import grpc 13 | import pytest 14 | import ratings_api.ratings_features_user_pb2 as pb2 15 | import ratings_api.ratings_features_user_pb2_grpc as pb2_grpc 16 | import yaml 17 | from pytest import mark 18 | from pytest_operator.plugin import OpsTest 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) 23 | RATINGS = "ratings" 24 | DB = "db" 25 | TRAEFIK = "traefik" 26 | 27 | 28 | @pytest.mark.abort_on_fail 29 | async def test_build_and_deploy(ops_test: OpsTest): 30 | """Build the charm-under-test and deploy it together with related charms. 31 | 32 | Assert on the unit status before any relations/configurations take place. 33 | """ 34 | # Build and deploy charm from local source folder 35 | charm = await ops_test.build_charm(".") 36 | resources = {"ratings-image": METADATA["resources"]["ratings-image"]["upstream-source"]} 37 | 38 | # Deploy the charm and wait for active/idle status 39 | await asyncio.gather( 40 | ops_test.model.deploy(charm, resources=resources, application_name=RATINGS, trust=True), 41 | ops_test.model.wait_for_idle( 42 | apps=[RATINGS], status="waiting", raise_on_blocked=True, timeout=1000 43 | ), 44 | ) 45 | 46 | 47 | @mark.abort_on_fail 48 | async def test_database_relation(ops_test: OpsTest): 49 | """Test that the charm can be successfully related to PostgreSQL.""" 50 | await asyncio.gather( 51 | ops_test.model.deploy( 52 | "postgresql-k8s", channel="14/edge", application_name=DB, trust=True 53 | ), 54 | ops_test.model.wait_for_idle( 55 | apps=[DB], status="active", raise_on_blocked=True, timeout=1000 56 | ), 57 | ) 58 | 59 | await asyncio.gather( 60 | ops_test.model.integrate(RATINGS, DB), 61 | ops_test.model.wait_for_idle( 62 | apps=[RATINGS], status="active", raise_on_blocked=True, timeout=1000 63 | ), 64 | ) 65 | 66 | 67 | @mark.abort_on_fail 68 | async def test_ratings_scale(ops_test: OpsTest): 69 | """Test that the charm can be scaled out and still talk to the database.""" 70 | await asyncio.gather( 71 | ops_test.model.applications[RATINGS].scale(2), 72 | ops_test.model.wait_for_idle( 73 | apps=[RATINGS], 74 | status="active", 75 | timeout=1000, 76 | wait_for_exact_units=2, 77 | ), 78 | ) 79 | 80 | 81 | @mark.abort_on_fail 82 | async def test_ratings_authenticate_user(ops_test: OpsTest): 83 | """End-to-end test to ensure the app can interact with the database.""" 84 | status = await ops_test.model.get_status() # noqa: F821 85 | address = status["applications"][RATINGS]["public-address"] 86 | 87 | channel = grpc.insecure_channel(f"{address}:18080") 88 | stub = pb2_grpc.UserStub(channel) 89 | message = pb2.AuthenticateRequest(id=secrets.token_hex(32)) 90 | response = stub.Authenticate(message) 91 | assert response.token 92 | 93 | 94 | @pytest.mark.abort_on_fail 95 | async def test_ingress_traefik_k8s(ops_test): 96 | """Test that the charm can be integrated with the Traefik ingress.""" 97 | await asyncio.gather( 98 | ops_test.model.deploy( 99 | "traefik-k8s", 100 | application_name=TRAEFIK, 101 | channel="stable", 102 | config={"routing_mode": "subdomain", "external_hostname": "foo.bar"}, 103 | trust=True, 104 | ), 105 | ops_test.model.wait_for_idle(apps=[TRAEFIK], status="active", timeout=1000), 106 | ) 107 | 108 | # Create the relation 109 | await ops_test.model.integrate(f"{RATINGS}:ingress", TRAEFIK) 110 | # Wait for the two apps to quiesce 111 | await ops_test.model.wait_for_idle(apps=[RATINGS, TRAEFIK], status="active", timeout=1000) 112 | 113 | result = await _retrieve_proxied_endpoints(ops_test, TRAEFIK) 114 | assert result.get(RATINGS, None) == {"url": f"http://{ops_test.model_name}-{RATINGS}.foo.bar/"} 115 | 116 | 117 | @pytest.mark.skipif( 118 | (not os.environ.get("GITHUB_ACTION", "")), 119 | reason="""This test requires host configuration which might not be present on your machine. 120 | 121 | If you know what you're doing, run `export GITHUB_ACTION=Foo` or similar prior to running 122 | `tox -e integration`. 123 | """, 124 | ) 125 | async def test_ratings_register_user_through_ingress(ops_test: OpsTest): 126 | """End-to-end test to ensure the app can be interacted with from behind Traefik.""" 127 | address = f"{ops_test.model_name}-{RATINGS}.foo.bar:80" 128 | channel = grpc.insecure_channel(address) 129 | stub = pb2_grpc.UserStub(channel) 130 | message = pb2.AuthenticateRequest(id=secrets.token_hex(32)) 131 | response = stub.Authenticate(message) 132 | assert response.token 133 | 134 | 135 | async def _retrieve_proxied_endpoints(ops_test, traefik_application_name): 136 | traefik_application = ops_test.model.applications[traefik_application_name] 137 | traefik_first_unit = next(iter(traefik_application.units)) 138 | action = await traefik_first_unit.run_action("show-proxied-endpoints") 139 | await action.wait() 140 | result = await ops_test.model.get_action_output(action.id) 141 | 142 | return json.loads(result["proxied-endpoints"]) 143 | -------------------------------------------------------------------------------- /tests/unit/test_charm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Canonical 2 | # See LICENSE file for licensing details. 3 | 4 | import unittest 5 | from types import SimpleNamespace 6 | from unittest.mock import patch 7 | 8 | import ops 9 | import ops.testing 10 | from charm import RatingsCharm 11 | from ratings import Ratings 12 | 13 | DB_RELATION_DATA = { 14 | "database": "ratings", 15 | "endpoints": "postgres:5432", 16 | "password": "password", 17 | "username": "username", 18 | "version": "14.8", 19 | } 20 | 21 | MOCK_RATINGS = Ratings("postgres://username:password@postgres:5432/ratings", "deadbeef") 22 | 23 | 24 | class MockDatabaseEvent: 25 | def __init__(self, id, name="database"): 26 | self.name = name 27 | self.id = id 28 | 29 | 30 | class TestCharm(unittest.TestCase): 31 | def setUp(self): 32 | self.harness = ops.testing.Harness(RatingsCharm) 33 | self.addCleanup(self.harness.cleanup) 34 | self.harness.set_leader(True) 35 | self.harness.begin() 36 | 37 | def test_ratings_pebble_ready_no_relation(self): 38 | expected_plan = {} 39 | self.harness.container_pebble_ready("ratings") 40 | updated_plan = self.harness.get_container_pebble_plan("ratings").to_dict() 41 | self.assertEqual(expected_plan, updated_plan) 42 | self.assertEqual( 43 | self.harness.model.unit.status, ops.WaitingStatus("Waiting for database relation") 44 | ) 45 | 46 | def test_ratings_sets_database_name_on_database_relation(self): 47 | rel_id = self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 48 | self.harness.container_pebble_ready("ratings") 49 | app_data = self.harness.get_relation_data(rel_id, self.harness.charm.app.name) 50 | self.assertEqual(app_data, {"database": "ratings"}) 51 | 52 | def test_ratings_pebble_ready_waits_for_db_initialisation(self): 53 | self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 54 | self.harness.container_pebble_ready("ratings") 55 | self.assertEqual( 56 | self.harness.model.unit.status, ops.WaitingStatus("Ratings not yet initialised") 57 | ) 58 | 59 | @patch("charm.RatingsCharm._ratings", MOCK_RATINGS) 60 | @patch("ratings.Ratings.ready", lambda x: True) 61 | def test_ratings_pebble_ready_sets_correct_plan(self): 62 | self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 63 | self.harness.container_pebble_ready("ratings") 64 | self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) 65 | expected = { 66 | "services": { 67 | "ratings": { 68 | "override": "replace", 69 | "summary": "ratings", 70 | "command": "/bin/ratings", 71 | "startup": "enabled", 72 | "environment": { 73 | "APP_POSTGRES_URI": "postgres://username:password@postgres:5432/ratings", 74 | "APP_MIGRATION_POSTGRES_URI": "postgres://username:password@postgres:5432/ratings", 75 | "APP_JWT_SECRET": "deadbeef", 76 | "APP_LOG_LEVEL": "info", 77 | "APP_ENV": "dev", 78 | }, 79 | } 80 | }, 81 | } 82 | self.assertEqual(self.harness.get_container_pebble_plan("ratings").to_dict(), expected) 83 | 84 | @patch("charm.RatingsCharm._ratings", MOCK_RATINGS) 85 | @patch("ratings.Ratings.ready", lambda x: True) 86 | def test_ratings_pebble_ready_waits_for_container(self): 87 | self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 88 | self.harness.set_can_connect("ratings", False) 89 | self.harness.charm.on.ratings_pebble_ready.emit(SimpleNamespace(workload="foo")) 90 | self.assertEqual( 91 | self.harness.model.unit.status, ops.WaitingStatus("Waiting for ratings container") 92 | ) 93 | 94 | def test_ratings_database_created_ratings_not_initialised(self): 95 | rel_id = self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 96 | self.harness.charm._database.on.database_created.emit(MockDatabaseEvent(id=rel_id)) 97 | self.harness.set_can_connect("ratings", True) 98 | plan = self.harness.get_container_pebble_plan("ratings").to_dict() 99 | self.assertEqual(plan, {}) 100 | 101 | @patch("charm.DatabaseRequires.is_resource_created", lambda x: True) 102 | @patch("ratings.Ratings.ready", lambda x: True) 103 | def test_ratings_database_created_database_success(self): 104 | rel_id = self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 105 | self.harness.set_can_connect("ratings", True) 106 | self.harness.charm._database.on.database_created.emit(MockDatabaseEvent(id=rel_id)) 107 | self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) 108 | 109 | def test_ratings_db_connection_string_no_relation(self): 110 | self.assertEqual(self.harness.charm._db_connection_string(), "") 111 | 112 | @patch("charm.DatabaseRequires.fetch_relation_data", lambda x: {0: DB_RELATION_DATA}) 113 | def test_ratings_db_connection_string(self): 114 | self.harness.add_relation("database", "postgresql", unit_data=DB_RELATION_DATA) 115 | expected = "postgres://username:password@postgres:5432/ratings" 116 | self.assertEqual(self.harness.charm._db_connection_string(), expected) 117 | 118 | def test_ratings_jwt_secret_no_relation(self): 119 | new_secret = self.harness.charm._jwt_secret() 120 | self.assertEqual(new_secret, "") 121 | 122 | def test_ratings_jwt_secret_create(self): 123 | self.harness.add_relation("ratings-peers", "ubuntu-software-ratings") 124 | new_secret = self.harness.charm._jwt_secret() 125 | self.assertEqual(len(new_secret), 48) 126 | 127 | def test_ratings_jwt_secret_from_peer_data(self): 128 | content = {"jwt-secret": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"} 129 | secret_id = self.harness.add_model_secret(owner=self.harness.charm.app, content=content) 130 | self.harness.add_relation( 131 | "ratings-peers", "ubuntu-software-ratings", app_data={"jwt-secret-id": secret_id} 132 | ) 133 | secret = self.harness.charm._jwt_secret() 134 | self.assertEqual(secret, content["jwt-secret"]) 135 | 136 | def test_ratings_property_already_initialised(self): 137 | self.harness.charm._ratings_svc = "foobar" 138 | self.assertEqual(self.harness.charm._ratings, "foobar") 139 | 140 | @patch("charm.DatabaseRequires.is_resource_created", lambda x: True) 141 | @patch("charm.DatabaseRequires.fetch_relation_data", lambda x: {0: DB_RELATION_DATA}) 142 | def test_ratings_property_not_initialised(self): 143 | self.assertIsInstance(self.harness.charm._ratings, Ratings) 144 | -------------------------------------------------------------------------------- /src/charm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2024 Tim Holmes-Mitra 3 | # See LICENSE file for licensing details. 4 | 5 | import logging 6 | from typing import TYPE_CHECKING 7 | 8 | import ops 9 | from utils import get_or_fail, stringify 10 | from workload import WorkloadAgentBuilder, WorkloadAgentBuilderState 11 | 12 | if TYPE_CHECKING: # development import paths for type checking 13 | from lib.charms.data_platform_libs.v0.data_interfaces import DatabaseRequires 14 | from lib.charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider 15 | from lib.charms.loki_k8s.v0.loki_push_api import LogProxyConsumer 16 | from lib.charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider 17 | from lib.charms.traefik_route_k8s.v0.traefik_route import TraefikRouteRequirer 18 | 19 | else: # runtime import paths 20 | from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires # noqa: F401, I001 21 | from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider # noqa: F401, I001 22 | from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer # noqa: F401, I001 23 | from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider # noqa: F401, I001 24 | from charms.traefik_route_k8s.v0.traefik_route import TraefikRouteRequirer # noqa: F401, I001 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | class AppCenterRatings(ops.CharmBase): 30 | def __init__(self, *args) -> None: 31 | super().__init__(*args) 32 | 33 | builder = WorkloadAgentBuilder() 34 | builder.load_config_values(self.config) 35 | 36 | self._builder = builder 37 | self._container = self.unit.get_container("workload") 38 | 39 | # Inspired from https://github.com/canonical/grafana-k8s-operator/blob/main/src/charm.py#L177 40 | self._ingress = TraefikRouteRequirer( 41 | self, 42 | self.model.get_relation(builder.ingress_relation_name), # type: ignore 43 | builder.ingress_relation_name, 44 | ) 45 | 46 | self._db = DatabaseRequires( 47 | self, 48 | relation_name=get_or_fail(builder.db_relation_name), 49 | database_name=get_or_fail(builder.db_name), 50 | ) 51 | 52 | self._configure_observability() 53 | self._register_events() 54 | 55 | def _configure_observability(self) -> None: 56 | """Observability is non-blocking.""" 57 | builder = self._builder 58 | port = builder.port 59 | 60 | self._prometheus_scraping = MetricsEndpointProvider( 61 | self, 62 | relation_name=builder.metrics_relation_name, 63 | jobs=[{"static_configs": [{"targets": [f"*:{port}"]}]}], 64 | refresh_event=self.on.config_changed, 65 | ) 66 | 67 | self._logging = LogProxyConsumer( 68 | self, 69 | relation_name=builder.log_relation_name, 70 | log_files=[builder.log_file], 71 | ) 72 | 73 | self._grafana_dashboards = GrafanaDashboardProvider( 74 | self, relation_name=builder.grafana_relation_name 75 | ) 76 | 77 | def _register_events(self) -> None: 78 | observe = self.framework.observe 79 | 80 | observe(self.on.config_changed, self._on_config_changed) 81 | 82 | observe(self.on.workload_pebble_ready, self._try_start) 83 | 84 | observe(self._db.on.database_created, self._try_start) 85 | observe(self._db.on.endpoints_changed, self._try_start) 86 | observe(self.on.database_relation_broken, self._on_db_relation_broken) 87 | 88 | observe(self.on.ingress_relation_joined, self._try_start) 89 | observe(self._ingress.on.ready, self._try_start) 90 | observe(self.on.leader_elected, self._try_start) 91 | observe(self.on.config_changed, self._try_start) 92 | 93 | def _on_config_changed(self, event: ops.ConfigChangedEvent): 94 | env = self.config.get("env") 95 | 96 | if not env: 97 | self.unit.status = ops.BlockedStatus("Charm's env unset. Must be prod, stg, or local") 98 | return 99 | 100 | logger.info(f"Environment set to: {env}") 101 | self._builder.set_env(env) 102 | self._try_start(event) 103 | 104 | def _try_start(self, event: ops.EventBase) -> None: 105 | self.unit.status = ops.WaitingStatus("Trying to start workload") 106 | 107 | if not self._container.can_connect(): 108 | self.unit.status = ops.WaitingStatus("Cannot connect to container") 109 | return 110 | 111 | self._try_fetch_db_relation() 112 | self._try_configure_ingress(event) 113 | 114 | builder = self._builder 115 | builder_state = builder.get_state() 116 | 117 | if not builder_state == WorkloadAgentBuilderState.Ready: 118 | self.unit.status = ops.WaitingStatus(builder_state.name) 119 | logger.debug(stringify(builder)) 120 | return 121 | 122 | workload_agent = builder.build() 123 | 124 | try: 125 | ingress_config = workload_agent.create_ingress_config 126 | self._ingress.submit_to_traefik(ingress_config) 127 | 128 | layer = workload_agent.create_pebble_layer 129 | self._container.add_layer(workload_agent.name, layer, combine=True) 130 | self._container.replan() 131 | self.unit.open_port(protocol="tcp", port=workload_agent.port) 132 | 133 | version = workload_agent.fetch_version() 134 | self.unit.set_workload_version(version) 135 | 136 | self.unit.status = ops.ActiveStatus("🚀") 137 | 138 | except ops.pebble.ConnectionError as e: 139 | logger.error(f"Failed to connect to Pebble: {e}") 140 | self.unit.status = ops.BlockedStatus("Could not connect to container") 141 | 142 | except Exception as e: 143 | logger.error(f"Pebble replan failed: {e}") 144 | self.unit.status = ops.BlockedStatus("Failed to configure container") 145 | 146 | def _try_configure_ingress(self, event: ops.EventBase) -> None: 147 | if not self.unit.is_leader(): 148 | return 149 | 150 | # When self._ingress._relation is first set in __init__ it too early in 151 | # the charm's life. So, when we capture the relation from the event. 152 | if ( 153 | isinstance(event, ops.RelationJoinedEvent) 154 | and event.relation.name == self._builder.ingress_relation_name 155 | ): 156 | self._ingress._relation = event.relation 157 | 158 | self._builder.set_ingress_ready(self._ingress.is_ready()) 159 | 160 | def _try_fetch_db_relation(self) -> None: 161 | relations = self._db.fetch_relation_data() 162 | 163 | for data in relations.values(): 164 | if not data: 165 | continue 166 | 167 | host, port = data["endpoints"].split(":") 168 | ( 169 | self._builder.set_db_host(host) 170 | .set_db_port(int(port)) 171 | .set_db_username(data["username"]) 172 | .set_db_password(data["password"]) 173 | ) 174 | 175 | def _on_db_relation_broken(self, _: ops.EventBase | None = None) -> None: 176 | self.unit.status = ops.WaitingStatus("Db relation broken") 177 | 178 | 179 | if __name__ == "__main__": # pragma: nocover 180 | ops.main(AppCenterRatings) # type: ignore 181 | -------------------------------------------------------------------------------- /src/workload.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from dataclasses import dataclass 3 | from enum import Enum 4 | from typing import Optional 5 | 6 | import ops 7 | from entities import About, LogLevel, WorkloadEnv 8 | from utils import get_or_fail 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class WorkloadAgentBuilderState(Enum): 14 | DatabaseNotReady = "DatabaseNotReady" 15 | IngressNotReady = "IngressNotReady" 16 | EnvNotSet = "EnvNotSet" 17 | JwtSecretNotSet = "JwtSecretNotSet" 18 | Ready = "All config values set" 19 | 20 | 21 | @dataclass 22 | class WorkloadAgent: 23 | env: WorkloadEnv 24 | 25 | model: str 26 | name: str 27 | port: int 28 | log_level: LogLevel 29 | 30 | jwt_secret: str 31 | 32 | db_name: str 33 | db_relation_name: str 34 | db_host: str 35 | db_port: int 36 | db_username: str 37 | db_password: str 38 | 39 | @property 40 | def external_hostname(self) -> str: 41 | switch = { 42 | WorkloadEnv.Prod: "ratings.ubuntu.com", 43 | WorkloadEnv.Stg: "ratings.stg.ubuntu.com", 44 | WorkloadEnv.Local: "ratings.ubuntu.local", 45 | } 46 | 47 | return switch[self.env] 48 | 49 | @property 50 | def create_ingress_config(self) -> dict: 51 | routers = { 52 | self.name: { 53 | "rule": f"Host(`{self.external_hostname}`)", 54 | "service": f"{self.name}-service", 55 | "entryPoints": ["web"], 56 | } 57 | } 58 | 59 | services = { 60 | f"{self.name}-service": { 61 | "loadBalancer": { 62 | "servers": [ 63 | {"url": f"h2c://{self.name}.{self.model}.svc.cluster.local:{self.port}"} 64 | ] 65 | } 66 | } 67 | } 68 | 69 | return {"http": {"routers": routers, "services": services}} 70 | 71 | @property 72 | def create_pebble_layer(self) -> ops.pebble.Layer: 73 | db_connection_string = f"postgresql://{self.db_username}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}" 74 | 75 | environment: dict = { 76 | "APP_ENV": self.env.value, 77 | "APP_HOST": "0.0.0.0", 78 | "APP_LOG_LEVEL": self.log_level.value, 79 | "APP_JWT_SECRET": self.jwt_secret, 80 | "APP_NAME": self.name, 81 | "APP_PORT": self.port, 82 | "APP_POSTGRES_URI": db_connection_string, 83 | "APP_MIGRATION_POSTGRES_URI": db_connection_string, 84 | "APP_ADMIN_USER": "shadow", 85 | "APP_ADMIN_PASSWORD": "maria", 86 | } 87 | 88 | layer = ops.pebble.Layer( 89 | { 90 | "summary": "app-center-ratings pebble layer", 91 | "services": { 92 | self.name: { 93 | "override": "replace", 94 | "summary": f"{self.name} pebble config layer", 95 | "startup": "enabled", 96 | "command": "/app/ratings", 97 | "environment": environment, 98 | } 99 | }, 100 | } 101 | ) 102 | return layer 103 | 104 | def api_url(self, path="") -> str: 105 | return f"http://localhost:{self.port}/{path}" 106 | 107 | def fetch_version(self) -> str: 108 | about = About(version="0.0.1") 109 | return about.version 110 | 111 | 112 | class WorkloadAgentBuilder: 113 | """This is some context. 114 | 115 | - values that are known should be set 116 | - values that come from an event / relation should be wrapped in Optional 117 | - in theory, this is the only file that need be edited to take this template and create a new charm. 118 | """ 119 | 120 | def __init__(self) -> None: 121 | self.model = "desktop" 122 | self.env: Optional[WorkloadEnv] = None 123 | self.name = "app-ratings" 124 | self.port = 8080 125 | self.jwt_secret: Optional[str] 126 | 127 | self.db_name = "ratings" 128 | self.db_relation_name = "database" 129 | self.db_host: Optional[str] = None 130 | self.db_port: Optional[int] = None 131 | self.db_username: Optional[str] = None 132 | self.db_password: Optional[str] = None 133 | 134 | """The ingress provider is Traefik.""" 135 | self.ingress_relation_name = "ingress" 136 | self.ingress_ready = False 137 | 138 | """These options are to wire up the workload to the observability stack.""" 139 | self.log_level = LogLevel.Info 140 | self.log_file = "/var/log/workload.log" 141 | self.log_relation_name = "log-proxy" 142 | self.grafana_relation_name = "grafana-dashboard" 143 | self.metrics_relation_name = "metrics-endpoint" 144 | 145 | def load_config_values(self, config: ops.ConfigData) -> "WorkloadAgentBuilder": 146 | self.set_env(config.get("env", "")) 147 | self.set_jwt_secret(config.get("jwt_secret", "")) 148 | self.set_log_level(config.get("log_level", "")) 149 | return self 150 | 151 | def set_env(self, value: str) -> "WorkloadAgentBuilder": 152 | self.env = WorkloadEnv.try_from_string(value) 153 | return self 154 | 155 | def set_db_host(self, value: str) -> "WorkloadAgentBuilder": 156 | self.db_host = value 157 | return self 158 | 159 | def set_db_port(self, value: int) -> "WorkloadAgentBuilder": 160 | self.db_port = value 161 | return self 162 | 163 | def set_db_username(self, value: str) -> "WorkloadAgentBuilder": 164 | self.db_username = value 165 | return self 166 | 167 | def set_db_password(self, value: str) -> "WorkloadAgentBuilder": 168 | self.db_password = value 169 | return self 170 | 171 | def set_ingress_ready(self, value: bool = True) -> "WorkloadAgentBuilder": 172 | self.ingress_ready = value 173 | return self 174 | 175 | def set_log_level(self, value: str) -> "WorkloadAgentBuilder": 176 | self.log_level = LogLevel.try_from_string(value) 177 | return self 178 | 179 | def set_jwt_secret(self, value: str) -> "WorkloadAgentBuilder": 180 | self.jwt_secret = value 181 | return self 182 | 183 | def get_state(self) -> WorkloadAgentBuilderState: 184 | db_ready = all( 185 | value is not None 186 | for value in [ 187 | self.db_host, 188 | self.db_port, 189 | self.db_username, 190 | self.db_password, 191 | ] 192 | ) 193 | 194 | if not db_ready: 195 | return WorkloadAgentBuilderState.DatabaseNotReady 196 | 197 | if not self.jwt_secret: 198 | return WorkloadAgentBuilderState.JwtSecretNotSet 199 | 200 | if not self.env: 201 | return WorkloadAgentBuilderState.EnvNotSet 202 | 203 | if not self.ingress_ready: 204 | return WorkloadAgentBuilderState.IngressNotReady 205 | 206 | return WorkloadAgentBuilderState.Ready 207 | 208 | def build(self) -> WorkloadAgent: 209 | return WorkloadAgent( 210 | env=get_or_fail(self.env, "env"), 211 | model=self.model, 212 | name=self.name, 213 | port=self.port, 214 | jwt_secret=get_or_fail(self.jwt_secret, "jwt_secret"), 215 | log_level=self.log_level, 216 | db_name=self.db_name, 217 | db_relation_name=self.db_relation_name, 218 | db_host=get_or_fail(self.db_host, "db_host"), 219 | db_port=get_or_fail(self.db_port, "db_port"), 220 | db_username=get_or_fail(self.db_username, "db_username"), 221 | db_password=get_or_fail(self.db_password, "db_password"), 222 | ) 223 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023 Canonical 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /lib/charms/traefik_route_k8s/v0/traefik_route.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2022 Canonical Ltd. 3 | # See LICENSE file for licensing details. 4 | 5 | r"""# Interface Library for traefik_route. 6 | 7 | This library wraps relation endpoints for traefik_route. The requirer of this 8 | relation is the traefik-route-k8s charm, or any charm capable of providing 9 | Traefik configuration files. The provider is the traefik-k8s charm, or another 10 | charm willing to consume Traefik configuration files. 11 | 12 | ## Getting Started 13 | 14 | To get started using the library, you just need to fetch the library using `charmcraft`. 15 | 16 | ```shell 17 | cd some-charm 18 | charmcraft fetch-lib charms.traefik_route_k8s.v0.traefik_route 19 | ``` 20 | 21 | To use the library from the provider side (Traefik): 22 | 23 | ```yaml 24 | requires: 25 | traefik_route: 26 | interface: traefik_route 27 | limit: 1 28 | ``` 29 | 30 | ```python 31 | from charms.traefik_route_k8s.v0.traefik_route import TraefikRouteProvider 32 | 33 | class TraefikCharm(CharmBase): 34 | def __init__(self, *args): 35 | # ... 36 | self.traefik_route = TraefikRouteProvider(self) 37 | 38 | self.framework.observe( 39 | self.traefik_route.on.ready, self._handle_traefik_route_ready 40 | ) 41 | 42 | def _handle_traefik_route_ready(self, event): 43 | config: str = self.traefik_route.get_config(event.relation) # yaml 44 | # use config to configure Traefik 45 | ``` 46 | 47 | To use the library from the requirer side (TraefikRoute): 48 | 49 | ```yaml 50 | requires: 51 | traefik-route: 52 | interface: traefik_route 53 | limit: 1 54 | optional: false 55 | ``` 56 | 57 | ```python 58 | # ... 59 | from charms.traefik_route_k8s.v0.traefik_route import TraefikRouteRequirer 60 | 61 | class TraefikRouteCharm(CharmBase): 62 | def __init__(self, *args): 63 | # ... 64 | traefik_route = TraefikRouteRequirer( 65 | self, self.model.relations.get("traefik-route"), 66 | "traefik-route" 67 | ) 68 | if traefik_route.is_ready(): 69 | traefik_route.submit_to_traefik( 70 | config={'my': {'traefik': 'configuration'}} 71 | ) 72 | 73 | ``` 74 | """ 75 | import logging 76 | from typing import Optional 77 | 78 | import yaml 79 | from ops.charm import CharmBase, CharmEvents, RelationEvent 80 | from ops.framework import EventSource, Object, StoredState 81 | from ops.model import Relation 82 | 83 | # The unique Charmhub library identifier, never change it 84 | LIBID = "fe2ac43a373949f2bf61383b9f35c83c" 85 | 86 | # Increment this major API version when introducing breaking changes 87 | LIBAPI = 0 88 | 89 | # Increment this PATCH version before using `charmcraft publish-lib` or reset 90 | # to 0 if you are raising the major API version 91 | LIBPATCH = 9 92 | 93 | log = logging.getLogger(__name__) 94 | 95 | 96 | class TraefikRouteException(RuntimeError): 97 | """Base class for exceptions raised by TraefikRoute.""" 98 | 99 | 100 | class UnauthorizedError(TraefikRouteException): 101 | """Raised when the unit needs leadership to perform some action.""" 102 | 103 | 104 | class TraefikRouteProviderReadyEvent(RelationEvent): 105 | """Event emitted when Traefik is ready to provide ingress for a routed unit.""" 106 | 107 | 108 | class TraefikRouteProviderDataRemovedEvent(RelationEvent): 109 | """Event emitted when a routed ingress relation is removed.""" 110 | 111 | 112 | class TraefikRouteRequirerReadyEvent(RelationEvent): 113 | """Event emitted when a unit requesting ingress has provided all data Traefik needs.""" 114 | 115 | 116 | class TraefikRouteRequirerEvents(CharmEvents): 117 | """Container for TraefikRouteRequirer events.""" 118 | 119 | ready = EventSource(TraefikRouteRequirerReadyEvent) 120 | 121 | 122 | class TraefikRouteProviderEvents(CharmEvents): 123 | """Container for TraefikRouteProvider events.""" 124 | 125 | ready = EventSource(TraefikRouteProviderReadyEvent) # TODO rename to data_provided in v1 126 | data_removed = EventSource(TraefikRouteProviderDataRemovedEvent) 127 | 128 | 129 | class TraefikRouteProvider(Object): 130 | """Implementation of the provider of traefik_route. 131 | 132 | This will presumably be owned by a Traefik charm. 133 | The main idea is that Traefik will observe the `ready` event and, upon 134 | receiving it, will fetch the config from the TraefikRoute's application databag, 135 | apply it, and update its own app databag to let Route know that the ingress 136 | is there. 137 | The TraefikRouteProvider provides api to do this easily. 138 | """ 139 | 140 | on = TraefikRouteProviderEvents() # pyright: ignore 141 | _stored = StoredState() 142 | 143 | def __init__( 144 | self, 145 | charm: CharmBase, 146 | relation_name: str = "traefik-route", 147 | external_host: str = "", 148 | *, 149 | scheme: str = "http", 150 | ): 151 | """Constructor for TraefikRouteProvider. 152 | 153 | Args: 154 | charm: The charm that is instantiating the instance. 155 | relation_name: The name of the relation relation_name to bind to 156 | (defaults to "traefik-route"). 157 | external_host: The external host. 158 | scheme: The scheme. 159 | """ 160 | super().__init__(charm, relation_name) 161 | self._stored.set_default(external_host=None, scheme=None) 162 | 163 | self._charm = charm 164 | self._relation_name = relation_name 165 | 166 | if ( 167 | self._stored.external_host != external_host # pyright: ignore 168 | or self._stored.scheme != scheme # pyright: ignore 169 | ): 170 | # If traefik endpoint details changed, update 171 | self.update_traefik_address(external_host=external_host, scheme=scheme) 172 | 173 | self.framework.observe( 174 | self._charm.on[relation_name].relation_changed, self._on_relation_changed 175 | ) 176 | self.framework.observe( 177 | self._charm.on[relation_name].relation_broken, self._on_relation_broken 178 | ) 179 | 180 | @property 181 | def external_host(self) -> str: 182 | """Return the external host set by Traefik, if any.""" 183 | self._update_stored() 184 | return self._stored.external_host or "" # type: ignore 185 | 186 | @property 187 | def scheme(self) -> str: 188 | """Return the scheme set by Traefik, if any.""" 189 | self._update_stored() 190 | return self._stored.scheme or "" # type: ignore 191 | 192 | @property 193 | def relations(self): 194 | """The list of Relation instances associated with this endpoint.""" 195 | return list(self._charm.model.relations[self._relation_name]) 196 | 197 | def _update_stored(self) -> None: 198 | """Ensure that the stored data is up-to-date. 199 | 200 | This is split out into a separate method since, in the case of multi-unit deployments, 201 | removal of a `TraefikRouteRequirer` will not cause a `RelationEvent`, but the guard on 202 | app data ensures that only the previous leader will know what it is. Separating it 203 | allows for reuse both when the property is called and if the relation changes, so a 204 | leader change where the new leader checks the property will do the right thing. 205 | """ 206 | if not self._charm.unit.is_leader(): 207 | return 208 | 209 | for relation in self._charm.model.relations[self._relation_name]: 210 | if not relation.app: 211 | self._stored.external_host = "" 212 | self._stored.scheme = "" 213 | return 214 | external_host = relation.data[relation.app].get("external_host", "") 215 | self._stored.external_host = ( 216 | external_host or self._stored.external_host # pyright: ignore 217 | ) 218 | scheme = relation.data[relation.app].get("scheme", "") 219 | self._stored.scheme = scheme or self._stored.scheme # pyright: ignore 220 | 221 | def _on_relation_changed(self, event: RelationEvent): 222 | if self.is_ready(event.relation): 223 | # todo check data is valid here? 224 | self.update_traefik_address() 225 | self.on.ready.emit(event.relation) 226 | 227 | def _on_relation_broken(self, event: RelationEvent): 228 | self.on.data_removed.emit(event.relation) 229 | 230 | def update_traefik_address( 231 | self, *, external_host: Optional[str] = None, scheme: Optional[str] = None 232 | ): 233 | """Ensure that requirers know the external host for Traefik.""" 234 | if not self._charm.unit.is_leader(): 235 | return 236 | 237 | for relation in self._charm.model.relations[self._relation_name]: 238 | relation.data[self._charm.app]["external_host"] = external_host or self.external_host 239 | relation.data[self._charm.app]["scheme"] = scheme or self.scheme 240 | 241 | # We first attempt to write relation data (which may raise) and only then update stored 242 | # state. 243 | self._stored.external_host = external_host 244 | self._stored.scheme = scheme 245 | 246 | @staticmethod 247 | def is_ready(relation: Relation) -> bool: 248 | """Whether TraefikRoute is ready on this relation. 249 | 250 | Returns True when the remote app shared the config; False otherwise. 251 | """ 252 | assert relation.app is not None # not currently handled anyway 253 | return "config" in relation.data[relation.app] 254 | 255 | @staticmethod 256 | def get_config(relation: Relation) -> Optional[str]: 257 | """Retrieve the config published by the remote application.""" 258 | # TODO: validate this config 259 | assert relation.app is not None # not currently handled anyway 260 | return relation.data[relation.app].get("config") 261 | 262 | 263 | class TraefikRouteRequirer(Object): 264 | """Wrapper for the requirer side of traefik-route. 265 | 266 | The traefik_route requirer will publish to the application databag an object like: 267 | { 268 | 'config': 269 | } 270 | 271 | NB: TraefikRouteRequirer does no validation; it assumes that the 272 | traefik-route-k8s charm will provide valid yaml-encoded config. 273 | The TraefikRouteRequirer provides api to store this config in the 274 | application databag. 275 | """ 276 | 277 | on = TraefikRouteRequirerEvents() # pyright: ignore 278 | _stored = StoredState() 279 | 280 | def __init__(self, charm: CharmBase, relation: Relation, relation_name: str = "traefik-route"): 281 | super(TraefikRouteRequirer, self).__init__(charm, relation_name) 282 | self._stored.set_default(external_host=None, scheme=None) 283 | 284 | self._charm = charm 285 | self._relation = relation 286 | 287 | self.framework.observe( 288 | self._charm.on[relation_name].relation_changed, self._on_relation_changed 289 | ) 290 | self.framework.observe( 291 | self._charm.on[relation_name].relation_broken, self._on_relation_broken 292 | ) 293 | 294 | @property 295 | def external_host(self) -> str: 296 | """Return the external host set by Traefik, if any.""" 297 | self._update_stored() 298 | return self._stored.external_host or "" # type: ignore 299 | 300 | @property 301 | def scheme(self) -> str: 302 | """Return the scheme set by Traefik, if any.""" 303 | self._update_stored() 304 | return self._stored.scheme or "" # type: ignore 305 | 306 | def _update_stored(self) -> None: 307 | """Ensure that the stored host is up-to-date. 308 | 309 | This is split out into a separate method since, in the case of multi-unit deployments, 310 | removal of a `TraefikRouteRequirer` will not cause a `RelationEvent`, but the guard on 311 | app data ensures that only the previous leader will know what it is. Separating it 312 | allows for reuse both when the property is called and if the relation changes, so a 313 | leader change where the new leader checks the property will do the right thing. 314 | """ 315 | if not self._charm.unit.is_leader(): 316 | return 317 | 318 | if self._relation: 319 | for relation in self._charm.model.relations[self._relation.name]: 320 | if not relation.app: 321 | self._stored.external_host = "" 322 | self._stored.scheme = "" 323 | return 324 | external_host = relation.data[relation.app].get("external_host", "") 325 | self._stored.external_host = ( 326 | external_host or self._stored.external_host # pyright: ignore 327 | ) 328 | scheme = relation.data[relation.app].get("scheme", "") 329 | self._stored.scheme = scheme or self._stored.scheme # pyright: ignore 330 | 331 | def _on_relation_changed(self, event: RelationEvent) -> None: 332 | """Update StoredState with external_host and other information from Traefik.""" 333 | self._update_stored() 334 | if self._charm.unit.is_leader(): 335 | self.on.ready.emit(event.relation) 336 | 337 | def _on_relation_broken(self, event: RelationEvent) -> None: 338 | """On RelationBroken, clear the stored data if set and emit an event.""" 339 | self._stored.external_host = "" 340 | if self._charm.unit.is_leader(): 341 | self.on.ready.emit(event.relation) 342 | 343 | def is_ready(self) -> bool: 344 | """Is the TraefikRouteRequirer ready to submit data to Traefik?""" 345 | return self._relation is not None 346 | 347 | def submit_to_traefik(self, config): 348 | """Relay an ingress configuration data structure to traefik. 349 | 350 | This will publish to TraefikRoute's traefik-route relation databag 351 | the config traefik needs to route the units behind this charm. 352 | """ 353 | if not self._charm.unit.is_leader(): 354 | raise UnauthorizedError() 355 | 356 | app_databag = self._relation.data[self._charm.app] 357 | 358 | # Traefik thrives on yaml, feels pointless to talk json to Route 359 | app_databag["config"] = yaml.safe_dump(config) 360 | -------------------------------------------------------------------------------- /lib/charms/grafana_k8s/v0/grafana_dashboard.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Canonical Ltd. 2 | # See LICENSE file for licensing details. 3 | 4 | """## Overview. 5 | 6 | This document explains how to integrate with the Grafana charm 7 | for the purpose of providing a dashboard which can be used by 8 | end users. It also explains the structure of the data 9 | expected by the `grafana-dashboard` interface, and may provide a 10 | mechanism or reference point for providing a compatible interface 11 | or library by providing a definitive reference guide to the 12 | structure of relation data which is shared between the Grafana 13 | charm and any charm providing datasource information. 14 | 15 | ## Provider Library Usage 16 | 17 | The Grafana charm interacts with its dashboards using its charm 18 | library. The goal of this library is to be as simple to use as 19 | possible, and instantiation of the class with or without changing 20 | the default arguments provides a complete use case. For the simplest 21 | use case of a charm which bundles dashboards and provides a 22 | `provides: grafana-dashboard` interface, 23 | 24 | requires: 25 | grafana-dashboard: 26 | interface: grafana_dashboard 27 | 28 | creation of a `GrafanaDashboardProvider` object with the default arguments is 29 | sufficient. 30 | 31 | :class:`GrafanaDashboardProvider` expects that bundled dashboards should 32 | be included in your charm with a default path of: 33 | 34 | path/to/charm.py 35 | path/to/src/grafana_dashboards/*.{json|json.tmpl|.tmpl} 36 | 37 | Where the files are Grafana dashboard JSON data either from the 38 | Grafana marketplace, or directly exported from a Grafana instance. 39 | Refer to the [official docs](https://grafana.com/tutorials/provision-dashboards-and-data-sources/) 40 | for more information. 41 | 42 | When constructing a dashboard that is intended to be consumed by COS, make sure to use variables 43 | for your datasources, and name them "prometheusds" and "lokids". You can also use the following 44 | juju topology variables in your dashboards: $juju_model, $juju_model_uuid, $juju_application 45 | and $juju_unit. Note, however, that if metrics are coming via peripheral charms (scrape-config 46 | or cos-config) then topology labels would not exist. 47 | 48 | The default constructor arguments are: 49 | 50 | `charm`: `self` from the charm instantiating this library 51 | `relation_name`: grafana-dashboard 52 | `dashboards_path`: "/src/grafana_dashboards" 53 | 54 | If your configuration requires any changes from these defaults, they 55 | may be set from the class constructor. It may be instantiated as 56 | follows: 57 | 58 | from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider 59 | 60 | class FooCharm: 61 | def __init__(self, *args): 62 | super().__init__(*args, **kwargs) 63 | ... 64 | self.grafana_dashboard_provider = GrafanaDashboardProvider(self) 65 | ... 66 | 67 | The first argument (`self`) should be a reference to the parent (providing 68 | dashboards), as this charm's lifecycle events will be used to re-submit 69 | dashboard information if a charm is upgraded, the pod is restarted, or other. 70 | 71 | An instantiated `GrafanaDashboardProvider` validates that the path specified 72 | in the constructor (or the default) exists, reads the file contents, then 73 | compresses them with LZMA and adds them to the application relation data 74 | when a relation is established with Grafana. 75 | 76 | Provided dashboards will be checked by Grafana, and a series of dropdown menus 77 | providing the ability to select query targets by Juju Model, application instance, 78 | and unit will be added if they do not exist. 79 | 80 | To avoid requiring `jinja` in `GrafanaDashboardProvider` users, template validation 81 | and rendering occurs on the other side of the relation, and relation data in 82 | the form of: 83 | 84 | { 85 | "event": { 86 | "valid": `true|false`, 87 | "errors": [], 88 | } 89 | } 90 | 91 | Will be returned if rendering or validation fails. In this case, the 92 | `GrafanaDashboardProvider` object will emit a `dashboard_status_changed` event 93 | of the type :class:`GrafanaDashboardEvent`, which will contain information 94 | about the validation error. 95 | 96 | This information is added to the relation data for the charms as serialized JSON 97 | from a dict, with a structure of: 98 | ``` 99 | { 100 | "application": { 101 | "dashboards": { 102 | "uuid": a uuid generated to ensure a relation event triggers, 103 | "templates": { 104 | "file:{hash}": { 105 | "content": `{compressed_template_data}`, 106 | "charm": `charm.meta.name`, 107 | "juju_topology": { 108 | "model": `charm.model.name`, 109 | "model_uuid": `charm.model.uuid`, 110 | "application": `charm.app.name`, 111 | "unit": `charm.unit.name`, 112 | } 113 | }, 114 | "file:{other_file_hash}": { 115 | ... 116 | }, 117 | }, 118 | }, 119 | }, 120 | } 121 | ``` 122 | 123 | This is ingested by :class:`GrafanaDashboardConsumer`, and is sufficient for configuration. 124 | 125 | The [COS Configuration Charm](https://charmhub.io/cos-configuration-k8s) can be used to 126 | add dashboards which are not bundled with charms. 127 | 128 | ## Consumer Library Usage 129 | 130 | The `GrafanaDashboardConsumer` object may be used by Grafana 131 | charms to manage relations with available dashboards. For this 132 | purpose, a charm consuming Grafana dashboard information should do 133 | the following things: 134 | 135 | 1. Instantiate the `GrafanaDashboardConsumer` object by providing it a 136 | reference to the parent (Grafana) charm and, optionally, the name of 137 | the relation that the Grafana charm uses to interact with dashboards. 138 | This relation must confirm to the `grafana-dashboard` interface. 139 | 140 | For example a Grafana charm may instantiate the 141 | `GrafanaDashboardConsumer` in its constructor as follows 142 | 143 | from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardConsumer 144 | 145 | def __init__(self, *args): 146 | super().__init__(*args) 147 | ... 148 | self.grafana_dashboard_consumer = GrafanaDashboardConsumer(self) 149 | ... 150 | 151 | 2. A Grafana charm also needs to listen to the 152 | `GrafanaDashboardConsumer` events emitted by the `GrafanaDashboardConsumer` 153 | by adding itself as an observer for these events: 154 | 155 | self.framework.observe( 156 | self.grafana_source_consumer.on.sources_changed, 157 | self._on_dashboards_changed, 158 | ) 159 | 160 | Dashboards can be retrieved the :meth:`dashboards`: 161 | 162 | It will be returned in the format of: 163 | 164 | ``` 165 | [ 166 | { 167 | "id": unique_id, 168 | "relation_id": relation_id, 169 | "charm": the name of the charm which provided the dashboard, 170 | "content": compressed_template_data 171 | }, 172 | ] 173 | ``` 174 | 175 | The consuming charm should decompress the dashboard. 176 | """ 177 | 178 | import base64 179 | import hashlib 180 | import json 181 | import logging 182 | import lzma 183 | import os 184 | import platform 185 | import re 186 | import subprocess 187 | import tempfile 188 | import uuid 189 | from pathlib import Path 190 | from typing import Any, Dict, List, Optional, Tuple, Union 191 | 192 | import yaml 193 | from ops.charm import ( 194 | CharmBase, 195 | HookEvent, 196 | RelationBrokenEvent, 197 | RelationChangedEvent, 198 | RelationCreatedEvent, 199 | RelationEvent, 200 | RelationRole, 201 | ) 202 | from ops.framework import ( 203 | EventBase, 204 | EventSource, 205 | Object, 206 | ObjectEvents, 207 | StoredDict, 208 | StoredList, 209 | StoredState, 210 | ) 211 | from ops.model import Relation 212 | 213 | # The unique Charmhub library identifier, never change it 214 | LIBID = "c49eb9c7dfef40c7b6235ebd67010a3f" 215 | 216 | # Increment this major API version when introducing breaking changes 217 | LIBAPI = 0 218 | 219 | # Increment this PATCH version before using `charmcraft publish-lib` or reset 220 | # to 0 if you are raising the major API version 221 | 222 | LIBPATCH = 35 223 | 224 | logger = logging.getLogger(__name__) 225 | 226 | 227 | DEFAULT_RELATION_NAME = "grafana-dashboard" 228 | DEFAULT_PEER_NAME = "grafana" 229 | RELATION_INTERFACE_NAME = "grafana_dashboard" 230 | 231 | TOPOLOGY_TEMPLATE_DROPDOWNS = [ # type: ignore 232 | { 233 | "allValue": ".*", 234 | "datasource": "${prometheusds}", 235 | "definition": "label_values(up,juju_model)", 236 | "description": None, 237 | "error": None, 238 | "hide": 0, 239 | "includeAll": True, 240 | "label": "Juju model", 241 | "multi": True, 242 | "name": "juju_model", 243 | "query": { 244 | "query": "label_values(up,juju_model)", 245 | "refId": "StandardVariableQuery", 246 | }, 247 | "refresh": 1, 248 | "regex": "", 249 | "skipUrlSync": False, 250 | "sort": 0, 251 | "tagValuesQuery": "", 252 | "tags": [], 253 | "tagsQuery": "", 254 | "type": "query", 255 | "useTags": False, 256 | }, 257 | { 258 | "allValue": ".*", 259 | "datasource": "${prometheusds}", 260 | "definition": 'label_values(up{juju_model=~"$juju_model"},juju_model_uuid)', 261 | "description": None, 262 | "error": None, 263 | "hide": 0, 264 | "includeAll": True, 265 | "label": "Juju model uuid", 266 | "multi": True, 267 | "name": "juju_model_uuid", 268 | "query": { 269 | "query": 'label_values(up{juju_model=~"$juju_model"},juju_model_uuid)', 270 | "refId": "StandardVariableQuery", 271 | }, 272 | "refresh": 1, 273 | "regex": "", 274 | "skipUrlSync": False, 275 | "sort": 0, 276 | "tagValuesQuery": "", 277 | "tags": [], 278 | "tagsQuery": "", 279 | "type": "query", 280 | "useTags": False, 281 | }, 282 | { 283 | "allValue": ".*", 284 | "datasource": "${prometheusds}", 285 | "definition": 'label_values(up{juju_model=~"$juju_model",juju_model_uuid=~"$juju_model_uuid"},juju_application)', 286 | "description": None, 287 | "error": None, 288 | "hide": 0, 289 | "includeAll": True, 290 | "label": "Juju application", 291 | "multi": True, 292 | "name": "juju_application", 293 | "query": { 294 | "query": 'label_values(up{juju_model=~"$juju_model",juju_model_uuid=~"$juju_model_uuid"},juju_application)', 295 | "refId": "StandardVariableQuery", 296 | }, 297 | "refresh": 1, 298 | "regex": "", 299 | "skipUrlSync": False, 300 | "sort": 0, 301 | "tagValuesQuery": "", 302 | "tags": [], 303 | "tagsQuery": "", 304 | "type": "query", 305 | "useTags": False, 306 | }, 307 | { 308 | "allValue": ".*", 309 | "datasource": "${prometheusds}", 310 | "definition": 'label_values(up{juju_model=~"$juju_model",juju_model_uuid=~"$juju_model_uuid",juju_application=~"$juju_application"},juju_unit)', 311 | "description": None, 312 | "error": None, 313 | "hide": 0, 314 | "includeAll": True, 315 | "label": "Juju unit", 316 | "multi": True, 317 | "name": "juju_unit", 318 | "query": { 319 | "query": 'label_values(up{juju_model=~"$juju_model",juju_model_uuid=~"$juju_model_uuid",juju_application=~"$juju_application"},juju_unit)', 320 | "refId": "StandardVariableQuery", 321 | }, 322 | "refresh": 1, 323 | "regex": "", 324 | "skipUrlSync": False, 325 | "sort": 0, 326 | "tagValuesQuery": "", 327 | "tags": [], 328 | "tagsQuery": "", 329 | "type": "query", 330 | "useTags": False, 331 | }, 332 | ] 333 | 334 | DATASOURCE_TEMPLATE_DROPDOWNS = [ # type: ignore 335 | { 336 | "description": None, 337 | "error": None, 338 | "hide": 0, 339 | "includeAll": True, 340 | "label": "Prometheus datasource", 341 | "multi": True, 342 | "name": "prometheusds", 343 | "options": [], 344 | "query": "prometheus", 345 | "refresh": 1, 346 | "regex": "", 347 | "skipUrlSync": False, 348 | "type": "datasource", 349 | }, 350 | { 351 | "description": None, 352 | "error": None, 353 | "hide": 0, 354 | "includeAll": True, 355 | "label": "Loki datasource", 356 | "multi": True, 357 | "name": "lokids", 358 | "options": [], 359 | "query": "loki", 360 | "refresh": 1, 361 | "regex": "", 362 | "skipUrlSync": False, 363 | "type": "datasource", 364 | }, 365 | ] 366 | 367 | REACTIVE_CONVERTER = { # type: ignore 368 | "allValue": None, 369 | "datasource": "${prometheusds}", 370 | "definition": 'label_values(up{juju_model=~"$juju_model",juju_model_uuid=~"$juju_model_uuid",juju_application=~"$juju_application"},host)', 371 | "description": None, 372 | "error": None, 373 | "hide": 0, 374 | "includeAll": True, 375 | "label": "hosts", 376 | "multi": True, 377 | "name": "host", 378 | "options": [], 379 | "query": { 380 | "query": 'label_values(up{juju_model=~"$juju_model",juju_model_uuid=~"$juju_model_uuid",juju_application=~"$juju_application"},host)', 381 | "refId": "StandardVariableQuery", 382 | }, 383 | "refresh": 1, 384 | "regex": "", 385 | "skipUrlSync": False, 386 | "sort": 1, 387 | "tagValuesQuery": "", 388 | "tags": [], 389 | "tagsQuery": "", 390 | "type": "query", 391 | "useTags": False, 392 | } 393 | 394 | 395 | class RelationNotFoundError(Exception): 396 | """Raised if there is no relation with the given name.""" 397 | 398 | def __init__(self, relation_name: str): 399 | self.relation_name = relation_name 400 | self.message = "No relation named '{}' found".format(relation_name) 401 | 402 | super().__init__(self.message) 403 | 404 | 405 | class RelationInterfaceMismatchError(Exception): 406 | """Raised if the relation with the given name has a different interface.""" 407 | 408 | def __init__( 409 | self, 410 | relation_name: str, 411 | expected_relation_interface: str, 412 | actual_relation_interface: str, 413 | ): 414 | self.relation_name = relation_name 415 | self.expected_relation_interface = expected_relation_interface 416 | self.actual_relation_interface = actual_relation_interface 417 | self.message = ( 418 | "The '{}' relation has '{}' as " 419 | "interface rather than the expected '{}'".format( 420 | relation_name, actual_relation_interface, expected_relation_interface 421 | ) 422 | ) 423 | 424 | super().__init__(self.message) 425 | 426 | 427 | class RelationRoleMismatchError(Exception): 428 | """Raised if the relation with the given name has a different direction.""" 429 | 430 | def __init__( 431 | self, 432 | relation_name: str, 433 | expected_relation_role: RelationRole, 434 | actual_relation_role: RelationRole, 435 | ): 436 | self.relation_name = relation_name 437 | self.expected_relation_interface = expected_relation_role 438 | self.actual_relation_role = actual_relation_role 439 | self.message = "The '{}' relation has role '{}' rather than the expected '{}'".format( 440 | relation_name, repr(actual_relation_role), repr(expected_relation_role) 441 | ) 442 | 443 | super().__init__(self.message) 444 | 445 | 446 | class InvalidDirectoryPathError(Exception): 447 | """Raised if the grafana dashboards folder cannot be found or is otherwise invalid.""" 448 | 449 | def __init__( 450 | self, 451 | grafana_dashboards_absolute_path: str, 452 | message: str, 453 | ): 454 | self.grafana_dashboards_absolute_path = grafana_dashboards_absolute_path 455 | self.message = message 456 | 457 | super().__init__(self.message) 458 | 459 | 460 | def _resolve_dir_against_charm_path(charm: CharmBase, *path_elements: str) -> str: 461 | """Resolve the provided path items against the directory of the main file. 462 | 463 | Look up the directory of the charmed operator file being executed. This is normally 464 | going to be the charm.py file of the charm including this library. Then, resolve 465 | the provided path elements and return its absolute path. 466 | 467 | Raises: 468 | InvalidDirectoryPathError if the resolved path does not exist or it is not a directory 469 | 470 | """ 471 | charm_dir = Path(str(charm.charm_dir)) 472 | if not charm_dir.exists() or not charm_dir.is_dir(): 473 | # Operator Framework does not currently expose a robust 474 | # way to determine the top level charm source directory 475 | # that is consistent across deployed charms and unit tests 476 | # Hence for unit tests the current working directory is used 477 | # TODO: updated this logic when the following ticket is resolved 478 | # https://github.com/canonical/operator/issues/643 479 | charm_dir = Path(os.getcwd()) 480 | 481 | dir_path = charm_dir.absolute().joinpath(*path_elements) 482 | 483 | if not dir_path.exists(): 484 | raise InvalidDirectoryPathError(str(dir_path), "directory does not exist") 485 | if not dir_path.is_dir(): 486 | raise InvalidDirectoryPathError(str(dir_path), "is not a directory") 487 | 488 | return str(dir_path) 489 | 490 | 491 | def _validate_relation_by_interface_and_direction( 492 | charm: CharmBase, 493 | relation_name: str, 494 | expected_relation_interface: str, 495 | expected_relation_role: RelationRole, 496 | ) -> None: 497 | """Verifies that a relation has the necessary characteristics. 498 | 499 | Verifies that the `relation_name` provided: (1) exists in metadata.yaml, 500 | (2) declares as interface the interface name passed as `relation_interface` 501 | and (3) has the right "direction", i.e., it is a relation that `charm` 502 | provides or requires. 503 | 504 | Args: 505 | charm: a `CharmBase` object to scan for the matching relation. 506 | relation_name: the name of the relation to be verified. 507 | expected_relation_interface: the interface name to be matched by the 508 | relation named `relation_name`. 509 | expected_relation_role: whether the `relation_name` must be either 510 | provided or required by `charm`. 511 | 512 | Raises: 513 | RelationNotFoundError: If there is no relation in the charm's metadata.yaml 514 | named like the value of the `relation_name` argument. 515 | RelationInterfaceMismatchError: If the relation interface of the 516 | relation named as the provided `relation_name` argument does not 517 | match the `expected_relation_interface` argument. 518 | RelationRoleMismatchError: If the relation named as the provided `relation_name` 519 | argument has a different role than what is specified by the 520 | `expected_relation_role` argument. 521 | """ 522 | if relation_name not in charm.meta.relations: 523 | raise RelationNotFoundError(relation_name) 524 | 525 | relation = charm.meta.relations[relation_name] 526 | 527 | actual_relation_interface = relation.interface_name 528 | if actual_relation_interface and actual_relation_interface != expected_relation_interface: 529 | raise RelationInterfaceMismatchError( 530 | relation_name, expected_relation_interface, actual_relation_interface 531 | ) 532 | 533 | if expected_relation_role == RelationRole.provides: 534 | if relation_name not in charm.meta.provides: 535 | raise RelationRoleMismatchError( 536 | relation_name, RelationRole.provides, RelationRole.requires 537 | ) 538 | elif expected_relation_role == RelationRole.requires: 539 | if relation_name not in charm.meta.requires: 540 | raise RelationRoleMismatchError( 541 | relation_name, RelationRole.requires, RelationRole.provides 542 | ) 543 | else: 544 | raise Exception("Unexpected RelationDirection: {}".format(expected_relation_role)) 545 | 546 | 547 | def _encode_dashboard_content(content: Union[str, bytes]) -> str: 548 | if isinstance(content, str): 549 | content = bytes(content, "utf-8") 550 | 551 | return base64.b64encode(lzma.compress(content)).decode("utf-8") 552 | 553 | 554 | def _decode_dashboard_content(encoded_content: str) -> str: 555 | return lzma.decompress(base64.b64decode(encoded_content.encode("utf-8"))).decode() 556 | 557 | 558 | def _convert_dashboard_fields(content: str, inject_dropdowns: bool = True) -> str: 559 | """Make sure values are present for Juju topology. 560 | 561 | Inserts Juju topology variables and selectors into the template, as well as 562 | a variable for Prometheus. 563 | """ 564 | dict_content = json.loads(content) 565 | datasources = {} 566 | existing_templates = False 567 | 568 | template_dropdowns = ( 569 | TOPOLOGY_TEMPLATE_DROPDOWNS + DATASOURCE_TEMPLATE_DROPDOWNS # type: ignore 570 | if inject_dropdowns 571 | else DATASOURCE_TEMPLATE_DROPDOWNS 572 | ) 573 | 574 | # If the dashboard has __inputs, get the names to replace them. These are stripped 575 | # from reactive dashboards in GrafanaDashboardAggregator, but charm authors in 576 | # newer charms may import them directly from the marketplace 577 | if "__inputs" in dict_content: 578 | for field in dict_content["__inputs"]: 579 | if "type" in field and field["type"] == "datasource": 580 | datasources[field["name"]] = field["pluginName"].lower() 581 | del dict_content["__inputs"] 582 | 583 | # If no existing template variables exist, just insert our own 584 | if "templating" not in dict_content: 585 | dict_content["templating"] = {"list": list(template_dropdowns)} # type: ignore 586 | else: 587 | # Otherwise, set a flag so we can go back later 588 | existing_templates = True 589 | for template_value in dict_content["templating"]["list"]: 590 | # Build a list of `datasource_name`: `datasource_type` mappings 591 | # The "query" field is actually "prometheus", "loki", "influxdb", etc 592 | if "type" in template_value and template_value["type"] == "datasource": 593 | datasources[template_value["name"]] = template_value["query"].lower() 594 | 595 | # Put our own variables in the template 596 | for d in template_dropdowns: # type: ignore 597 | if d not in dict_content["templating"]["list"]: 598 | dict_content["templating"]["list"].insert(0, d) 599 | 600 | dict_content = _replace_template_fields(dict_content, datasources, existing_templates) 601 | return json.dumps(dict_content) 602 | 603 | 604 | def _replace_template_fields( # noqa: C901 605 | dict_content: dict, datasources: dict, existing_templates: bool 606 | ) -> dict: 607 | """Make templated fields get cleaned up afterwards. 608 | 609 | If existing datasource variables are present, try to substitute them. 610 | """ 611 | replacements = {"loki": "${lokids}", "prometheus": "${prometheusds}"} 612 | used_replacements = [] # type: List[str] 613 | 614 | # If any existing datasources match types we know, or we didn't find 615 | # any templating variables at all, template them. 616 | if datasources or not existing_templates: 617 | panels = dict_content.get("panels", {}) 618 | if panels: 619 | dict_content["panels"] = _template_panels( 620 | panels, replacements, used_replacements, existing_templates, datasources 621 | ) 622 | 623 | # Find panels nested under rows 624 | rows = dict_content.get("rows", {}) 625 | if rows: 626 | for row_idx, row in enumerate(rows): 627 | if "panels" in row.keys(): 628 | rows[row_idx]["panels"] = _template_panels( 629 | row["panels"], 630 | replacements, 631 | used_replacements, 632 | existing_templates, 633 | datasources, 634 | ) 635 | 636 | dict_content["rows"] = rows 637 | 638 | # Finally, go back and pop off the templates we stubbed out 639 | deletions = [] 640 | for tmpl in dict_content["templating"]["list"]: 641 | if tmpl["name"] and tmpl["name"] in used_replacements: 642 | deletions.append(tmpl) 643 | 644 | for d in deletions: 645 | dict_content["templating"]["list"].remove(d) 646 | 647 | return dict_content 648 | 649 | 650 | def _template_panels( 651 | panels: dict, 652 | replacements: dict, 653 | used_replacements: list, 654 | existing_templates: bool, 655 | datasources: dict, 656 | ) -> dict: 657 | """Iterate through a `panels` object and template it appropriately.""" 658 | # Go through all the panels. If they have a datasource set, AND it's one 659 | # that we can convert to ${lokids} or ${prometheusds}, by stripping off the 660 | # ${} templating and comparing the name to the list we built, replace it, 661 | # otherwise, leave it alone. 662 | # 663 | for panel in panels: 664 | if "datasource" not in panel or not panel.get("datasource"): 665 | continue 666 | if not existing_templates: 667 | datasource = panel.get("datasource") 668 | if isinstance(datasource, str): 669 | if "loki" in datasource: 670 | panel["datasource"] = "${lokids}" 671 | elif "grafana" in datasource: 672 | continue 673 | else: 674 | panel["datasource"] = "${prometheusds}" 675 | elif isinstance(datasource, dict): 676 | # In dashboards exported by Grafana 9, datasource type is dict 677 | dstype = datasource.get("type", "") 678 | if dstype == "loki": 679 | panel["datasource"]["uid"] = "${lokids}" 680 | elif dstype == "prometheus": 681 | panel["datasource"]["uid"] = "${prometheusds}" 682 | else: 683 | logger.debug("Unrecognized datasource type '%s'; skipping", dstype) 684 | continue 685 | else: 686 | logger.error("Unknown datasource format: skipping") 687 | continue 688 | else: 689 | if isinstance(panel["datasource"], str): 690 | if panel["datasource"].lower() in replacements.values(): 691 | # Already a known template variable 692 | continue 693 | # Strip out variable characters and maybe braces 694 | ds = re.sub(r"(\$|\{|\})", "", panel["datasource"]) 695 | 696 | if ds not in datasources.keys(): 697 | # Unknown, non-templated datasource, potentially a Grafana builtin 698 | continue 699 | 700 | replacement = replacements.get(datasources[ds], "") 701 | if replacement: 702 | used_replacements.append(ds) 703 | panel["datasource"] = replacement or panel["datasource"] 704 | elif isinstance(panel["datasource"], dict): 705 | dstype = panel["datasource"].get("type", "") 706 | if panel["datasource"].get("uid", "").lower() in replacements.values(): 707 | # Already a known template variable 708 | continue 709 | # Strip out variable characters and maybe braces 710 | ds = re.sub(r"(\$|\{|\})", "", panel["datasource"].get("uid", "")) 711 | 712 | if ds not in datasources.keys(): 713 | # Unknown, non-templated datasource, potentially a Grafana builtin 714 | continue 715 | 716 | replacement = replacements.get(datasources[ds], "") 717 | if replacement: 718 | used_replacements.append(ds) 719 | panel["datasource"]["uid"] = replacement 720 | else: 721 | logger.error("Unknown datasource format: skipping") 722 | continue 723 | return panels 724 | 725 | 726 | def _inject_labels(content: str, topology: dict, transformer: "CosTool") -> str: 727 | """Inject Juju topology into panel expressions via CosTool. 728 | 729 | A dashboard will have a structure approximating: 730 | { 731 | "__inputs": [], 732 | "templating": { 733 | "list": [ 734 | { 735 | "name": "prometheusds", 736 | "type": "prometheus" 737 | } 738 | ] 739 | }, 740 | "panels": [ 741 | { 742 | "foo": "bar", 743 | "targets": [ 744 | { 745 | "some": "field", 746 | "expr": "up{job="foo"}" 747 | }, 748 | { 749 | "some_other": "field", 750 | "expr": "sum(http_requests_total{instance="$foo"}[5m])} 751 | } 752 | ], 753 | "datasource": "${someds}" 754 | } 755 | ] 756 | } 757 | 758 | `templating` is used elsewhere in this library, but the structure is not rigid. It is 759 | not guaranteed that a panel will actually have any targets (it could be a "spacer" with 760 | no datasource, hence no expression). It could have only one target. It could have multiple 761 | targets. It could have multiple targets of which only one has an `expr` to evaluate. We need 762 | to try to handle all of these concisely. 763 | 764 | `cos-tool` (`github.com/canonical/cos-tool` as a Go module in general) 765 | does not know "Grafana-isms", such as using `[$_variable]` to modify the query from the user 766 | interface, so we add placeholders (as `5y`, since it must parse, but a dashboard looking for 767 | five years for a panel query would be unusual). 768 | 769 | Args: 770 | content: dashboard content as a string 771 | topology: a dict containing topology values 772 | transformer: a 'CosTool' instance 773 | Returns: 774 | dashboard content with replaced values. 775 | """ 776 | dict_content = json.loads(content) 777 | 778 | if "panels" not in dict_content.keys(): 779 | return json.dumps(dict_content) 780 | 781 | # Go through all the panels and inject topology labels 782 | # Panels may have more than one 'target' where the expressions live, so that must be 783 | # accounted for. Additionally, `promql-transform` does not necessarily gracefully handle 784 | # expressions with range queries including variables. Exclude these. 785 | # 786 | # It is not a certainty that the `datasource` field will necessarily reflect the type, so 787 | # operate on all fields. 788 | panels = dict_content["panels"] 789 | topology_with_prefix = {"juju_{}".format(k): v for k, v in topology.items()} 790 | 791 | # We need to use an index so we can insert the changed element back later 792 | for panel_idx, panel in enumerate(panels): 793 | if not isinstance(panel, dict): 794 | continue 795 | 796 | # Use the index to insert it back in the same location 797 | panels[panel_idx] = _modify_panel(panel, topology_with_prefix, transformer) 798 | 799 | return json.dumps(dict_content) 800 | 801 | 802 | def _modify_panel(panel: dict, topology: dict, transformer: "CosTool") -> dict: 803 | """Inject Juju topology into panel expressions via CosTool. 804 | 805 | Args: 806 | panel: a dashboard panel as a dict 807 | topology: a dict containing topology values 808 | transformer: a 'CosTool' instance 809 | Returns: 810 | the panel with injected values 811 | """ 812 | if "targets" not in panel.keys(): 813 | return panel 814 | 815 | # Pre-compile a regular expression to grab values from inside of [] 816 | range_re = re.compile(r"\[(?P.*?)\]") 817 | # Do the same for any offsets 818 | offset_re = re.compile(r"offset\s+(?P-?\s*[$\w]+)") 819 | 820 | known_datasources = {"${prometheusds}": "promql", "${lokids}": "logql"} 821 | 822 | targets = panel["targets"] 823 | 824 | # We need to use an index so we can insert the changed element back later 825 | for idx, target in enumerate(targets): 826 | # If there's no expression, we don't need to do anything 827 | if "expr" not in target.keys(): 828 | continue 829 | expr = target["expr"] 830 | 831 | if "datasource" not in panel.keys(): 832 | continue 833 | 834 | if isinstance(panel["datasource"], str): 835 | if panel["datasource"] not in known_datasources: 836 | continue 837 | querytype = known_datasources[panel["datasource"]] 838 | elif isinstance(panel["datasource"], dict): 839 | if panel["datasource"]["uid"] not in known_datasources: 840 | continue 841 | querytype = known_datasources[panel["datasource"]["uid"]] 842 | else: 843 | logger.error("Unknown datasource format: skipping") 844 | continue 845 | 846 | # Capture all values inside `[]` into a list which we'll iterate over later to 847 | # put them back in-order. Then apply the regex again and replace everything with 848 | # `[5y]` so promql/parser will take it. 849 | # 850 | # Then do it again for offsets 851 | range_values = [m.group("value") for m in range_re.finditer(expr)] 852 | expr = range_re.sub(r"[5y]", expr) 853 | 854 | offset_values = [m.group("value") for m in offset_re.finditer(expr)] 855 | expr = offset_re.sub(r"offset 5y", expr) 856 | # Retrieve the new expression (which may be unchanged if there were no label 857 | # matchers in the expression, or if tt was unable to be parsed like logql. It's 858 | # virtually impossible to tell from any datasource "name" in a panel what the 859 | # actual type is without re-implementing a complete dashboard parser, but no 860 | # harm will some from passing invalid promql -- we'll just get the original back. 861 | # 862 | replacement = transformer.inject_label_matchers(expr, topology, querytype) 863 | 864 | if replacement == target["expr"]: 865 | # promql-tranform caught an error. Move on 866 | continue 867 | 868 | # Go back and substitute values in [] which were pulled out 869 | # Enumerate with an index... again. The same regex is ok, since it will still match 870 | # `[(.*?)]`, which includes `[5y]`, our placeholder 871 | for i, match in enumerate(range_re.finditer(replacement)): 872 | # Replace one-by-one, starting from the left. We build the string back with 873 | # `str.replace(string_to_replace, replacement_value, count)`. Limit the count 874 | # to one, since we are going through one-by-one through the list we saved earlier 875 | # in `range_values`. 876 | replacement = replacement.replace( 877 | "[{}]".format(match.group("value")), 878 | "[{}]".format(range_values[i]), 879 | 1, 880 | ) 881 | 882 | for i, match in enumerate(offset_re.finditer(replacement)): 883 | # Replace one-by-one, starting from the left. We build the string back with 884 | # `str.replace(string_to_replace, replacement_value, count)`. Limit the count 885 | # to one, since we are going through one-by-one through the list we saved earlier 886 | # in `range_values`. 887 | replacement = replacement.replace( 888 | "offset {}".format(match.group("value")), 889 | "offset {}".format(offset_values[i]), 890 | 1, 891 | ) 892 | 893 | # Use the index to insert it back in the same location 894 | targets[idx]["expr"] = replacement 895 | 896 | panel["targets"] = targets 897 | return panel 898 | 899 | 900 | def _type_convert_stored(obj): 901 | """Convert Stored* to their appropriate types, recursively.""" 902 | if isinstance(obj, StoredList): 903 | return list(map(_type_convert_stored, obj)) 904 | if isinstance(obj, StoredDict): 905 | rdict = {} # type: Dict[Any, Any] 906 | for k in obj.keys(): 907 | rdict[k] = _type_convert_stored(obj[k]) 908 | return rdict 909 | return obj 910 | 911 | 912 | class GrafanaDashboardsChanged(EventBase): 913 | """Event emitted when Grafana dashboards change.""" 914 | 915 | def __init__(self, handle, data=None): 916 | super().__init__(handle) 917 | self.data = data 918 | 919 | def snapshot(self) -> Dict: 920 | """Save grafana source information.""" 921 | return {"data": self.data} 922 | 923 | def restore(self, snapshot): 924 | """Restore grafana source information.""" 925 | self.data = snapshot["data"] 926 | 927 | 928 | class GrafanaDashboardEvents(ObjectEvents): 929 | """Events raised by :class:`GrafanaSourceEvents`.""" 930 | 931 | dashboards_changed = EventSource(GrafanaDashboardsChanged) 932 | 933 | 934 | class GrafanaDashboardEvent(EventBase): 935 | """Event emitted when Grafana dashboards cannot be resolved. 936 | 937 | Enables us to set a clear status on the provider. 938 | """ 939 | 940 | def __init__(self, handle, errors: List[Dict[str, str]] = [], valid: bool = False): 941 | super().__init__(handle) 942 | self.errors = errors 943 | self.error_message = "; ".join([error["error"] for error in errors if "error" in error]) 944 | self.valid = valid 945 | 946 | def snapshot(self) -> Dict: 947 | """Save grafana source information.""" 948 | return { 949 | "error_message": self.error_message, 950 | "valid": self.valid, 951 | "errors": json.dumps(self.errors), 952 | } 953 | 954 | def restore(self, snapshot): 955 | """Restore grafana source information.""" 956 | self.error_message = snapshot["error_message"] 957 | self.valid = snapshot["valid"] 958 | self.errors = json.loads(str(snapshot["errors"])) 959 | 960 | 961 | class GrafanaProviderEvents(ObjectEvents): 962 | """Events raised by :class:`GrafanaSourceEvents`.""" 963 | 964 | dashboard_status_changed = EventSource(GrafanaDashboardEvent) 965 | 966 | 967 | class GrafanaDashboardProvider(Object): 968 | """An API to provide Grafana dashboards to a Grafana charm.""" 969 | 970 | _stored = StoredState() 971 | on = GrafanaProviderEvents() # pyright: ignore 972 | 973 | def __init__( 974 | self, 975 | charm: CharmBase, 976 | relation_name: str = DEFAULT_RELATION_NAME, 977 | dashboards_path: str = "src/grafana_dashboards", 978 | ) -> None: 979 | """API to provide Grafana dashboard to a Grafana charmed operator. 980 | 981 | The :class:`GrafanaDashboardProvider` object provides an API 982 | to upload dashboards to a Grafana charm. In its most streamlined 983 | usage, the :class:`GrafanaDashboardProvider` is integrated in a 984 | charmed operator as follows: 985 | 986 | self.grafana = GrafanaDashboardProvider(self) 987 | 988 | The :class:`GrafanaDashboardProvider` will look for dashboard 989 | templates in the `/grafana_dashboards` folder. 990 | Additionally, dashboard templates can be uploaded programmatically 991 | via the :method:`GrafanaDashboardProvider.add_dashboard` method. 992 | 993 | To use the :class:`GrafanaDashboardProvider` API, you need a relation 994 | defined in your charm operator's metadata.yaml as follows: 995 | 996 | provides: 997 | grafana-dashboard: 998 | interface: grafana_dashboard 999 | 1000 | If you would like to use relation name other than `grafana-dashboard`, 1001 | you will need to specify the relation name via the `relation_name` 1002 | argument when instantiating the :class:`GrafanaDashboardProvider` object. 1003 | However, it is strongly advised to keep the default relation name, 1004 | so that people deploying your charm will have a consistent experience 1005 | with all other charms that provide Grafana dashboards. 1006 | 1007 | It is possible to provide a different file path for the Grafana dashboards 1008 | to be automatically managed by the :class:`GrafanaDashboardProvider` object 1009 | via the `dashboards_path` argument. This may be necessary when the directory 1010 | structure of your charmed operator repository is not the "usual" one as 1011 | generated by `charmcraft init`, for example when adding the charmed operator 1012 | in a Java repository managed by Maven or Gradle. However, unless there are 1013 | such constraints with other tooling, it is strongly advised to store the 1014 | Grafana dashboards in the default `/grafana_dashboards` 1015 | folder, in order to provide a consistent experience for other charmed operator 1016 | authors. 1017 | 1018 | Args: 1019 | charm: a :class:`CharmBase` object which manages this 1020 | :class:`GrafanaProvider` object. Generally this is 1021 | `self` in the instantiating class. 1022 | relation_name: a :string: name of the relation managed by this 1023 | :class:`GrafanaDashboardProvider`; it defaults to "grafana-dashboard". 1024 | dashboards_path: a filesystem path relative to the charm root 1025 | where dashboard templates can be located. By default, the library 1026 | expects dashboard files to be in the `/grafana_dashboards` 1027 | directory. 1028 | """ 1029 | _validate_relation_by_interface_and_direction( 1030 | charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.provides 1031 | ) 1032 | 1033 | try: 1034 | dashboards_path = _resolve_dir_against_charm_path(charm, dashboards_path) 1035 | except InvalidDirectoryPathError as e: 1036 | logger.warning( 1037 | "Invalid Grafana dashboards folder at %s: %s", 1038 | e.grafana_dashboards_absolute_path, 1039 | e.message, 1040 | ) 1041 | 1042 | super().__init__(charm, relation_name) 1043 | 1044 | self._charm = charm 1045 | self._relation_name = relation_name 1046 | self._dashboards_path = dashboards_path 1047 | 1048 | # No peer relation bucket we can rely on providers, keep StoredState here, too 1049 | self._stored.set_default(dashboard_templates={}) # type: ignore 1050 | 1051 | self.framework.observe(self._charm.on.leader_elected, self._update_all_dashboards_from_dir) 1052 | self.framework.observe(self._charm.on.upgrade_charm, self._update_all_dashboards_from_dir) 1053 | 1054 | self.framework.observe( 1055 | self._charm.on[self._relation_name].relation_created, 1056 | self._on_grafana_dashboard_relation_created, 1057 | ) 1058 | self.framework.observe( 1059 | self._charm.on[self._relation_name].relation_changed, 1060 | self._on_grafana_dashboard_relation_changed, 1061 | ) 1062 | 1063 | def add_dashboard(self, content: str, inject_dropdowns: bool = True) -> None: 1064 | """Add a dashboard to the relation managed by this :class:`GrafanaDashboardProvider`. 1065 | 1066 | Args: 1067 | content: a string representing a Jinja template. Currently, no 1068 | global variables are added to the Jinja template evaluation 1069 | context. 1070 | inject_dropdowns: a :boolean: indicating whether topology dropdowns should be 1071 | added to the dashboard 1072 | """ 1073 | # Update of storage must be done irrespective of leadership, so 1074 | # that the stored state is there when this unit becomes leader. 1075 | stored_dashboard_templates: Any = self._stored.dashboard_templates # pyright: ignore 1076 | 1077 | encoded_dashboard = _encode_dashboard_content(content) 1078 | 1079 | # Use as id the first chars of the encoded dashboard, so that 1080 | # it is predictable across units. 1081 | id = "prog:{}".format(encoded_dashboard[-24:-16]) 1082 | 1083 | stored_dashboard_templates[id] = self._content_to_dashboard_object( 1084 | encoded_dashboard, inject_dropdowns 1085 | ) 1086 | stored_dashboard_templates[id]["dashboard_alt_uid"] = self._generate_alt_uid(id) 1087 | 1088 | if self._charm.unit.is_leader(): 1089 | for dashboard_relation in self._charm.model.relations[self._relation_name]: 1090 | self._upset_dashboards_on_relation(dashboard_relation) 1091 | 1092 | def remove_non_builtin_dashboards(self) -> None: 1093 | """Remove all dashboards to the relation added via :method:`add_dashboard`.""" 1094 | # Update of storage must be done irrespective of leadership, so 1095 | # that the stored state is there when this unit becomes leader. 1096 | stored_dashboard_templates: Any = self._stored.dashboard_templates # pyright: ignore 1097 | 1098 | for dashboard_id in list(stored_dashboard_templates.keys()): 1099 | if dashboard_id.startswith("prog:"): 1100 | del stored_dashboard_templates[dashboard_id] 1101 | self._stored.dashboard_templates = stored_dashboard_templates 1102 | 1103 | if self._charm.unit.is_leader(): 1104 | for dashboard_relation in self._charm.model.relations[self._relation_name]: 1105 | self._upset_dashboards_on_relation(dashboard_relation) 1106 | 1107 | def update_dashboards(self) -> None: 1108 | """Trigger the re-evaluation of the data on all relations.""" 1109 | if self._charm.unit.is_leader(): 1110 | for dashboard_relation in self._charm.model.relations[self._relation_name]: 1111 | self._upset_dashboards_on_relation(dashboard_relation) 1112 | 1113 | def _update_all_dashboards_from_dir( 1114 | self, _: Optional[HookEvent] = None, inject_dropdowns: bool = True 1115 | ) -> None: 1116 | """Scans the built-in dashboards and updates relations with changes.""" 1117 | # Update of storage must be done irrespective of leadership, so 1118 | # that the stored state is there when this unit becomes leader. 1119 | 1120 | # Ensure we do not leave outdated dashboards by removing from stored all 1121 | # the encoded dashboards that start with "file/". 1122 | if self._dashboards_path: 1123 | stored_dashboard_templates: Any = self._stored.dashboard_templates # pyright: ignore 1124 | 1125 | for dashboard_id in list(stored_dashboard_templates.keys()): 1126 | if dashboard_id.startswith("file:"): 1127 | del stored_dashboard_templates[dashboard_id] 1128 | 1129 | # Path.glob uses fnmatch on the backend, which is pretty limited, so use a 1130 | # custom function for the filter 1131 | def _is_dashboard(p: Path) -> bool: 1132 | return p.is_file() and p.name.endswith((".json", ".json.tmpl", ".tmpl")) 1133 | 1134 | for path in filter(_is_dashboard, Path(self._dashboards_path).glob("*")): 1135 | # path = Path(path) 1136 | id = "file:{}".format(path.stem) 1137 | stored_dashboard_templates[id] = self._content_to_dashboard_object( 1138 | _encode_dashboard_content(path.read_bytes()), inject_dropdowns 1139 | ) 1140 | stored_dashboard_templates[id]["dashboard_alt_uid"] = self._generate_alt_uid(id) 1141 | 1142 | self._stored.dashboard_templates = stored_dashboard_templates 1143 | 1144 | if self._charm.unit.is_leader(): 1145 | for dashboard_relation in self._charm.model.relations[self._relation_name]: 1146 | self._upset_dashboards_on_relation(dashboard_relation) 1147 | 1148 | def _generate_alt_uid(self, key: str) -> str: 1149 | """Generate alternative uid for dashboards. 1150 | 1151 | Args: 1152 | key: A string used (along with charm.meta.name) to build the hash uid. 1153 | 1154 | Returns: A hash string. 1155 | """ 1156 | raw_dashboard_alt_uid = "{}-{}".format(self._charm.meta.name, key) 1157 | return hashlib.shake_256(raw_dashboard_alt_uid.encode("utf-8")).hexdigest(8) 1158 | 1159 | def _reinitialize_dashboard_data(self, inject_dropdowns: bool = True) -> None: 1160 | """Triggers a reload of dashboard outside of an eventing workflow. 1161 | 1162 | Args: 1163 | inject_dropdowns: a :bool: used to indicate whether topology dropdowns should be added 1164 | 1165 | This will destroy any existing relation data. 1166 | """ 1167 | try: 1168 | _resolve_dir_against_charm_path(self._charm, self._dashboards_path) 1169 | self._update_all_dashboards_from_dir(inject_dropdowns=inject_dropdowns) 1170 | 1171 | except InvalidDirectoryPathError as e: 1172 | logger.warning( 1173 | "Invalid Grafana dashboards folder at %s: %s", 1174 | e.grafana_dashboards_absolute_path, 1175 | e.message, 1176 | ) 1177 | stored_dashboard_templates: Any = self._stored.dashboard_templates # pyright: ignore 1178 | 1179 | for dashboard_id in list(stored_dashboard_templates.keys()): 1180 | if dashboard_id.startswith("file:"): 1181 | del stored_dashboard_templates[dashboard_id] 1182 | self._stored.dashboard_templates = stored_dashboard_templates 1183 | 1184 | # With all the file-based dashboards cleared out, force a refresh 1185 | # of relation data 1186 | if self._charm.unit.is_leader(): 1187 | for dashboard_relation in self._charm.model.relations[self._relation_name]: 1188 | self._upset_dashboards_on_relation(dashboard_relation) 1189 | 1190 | def _on_grafana_dashboard_relation_created(self, event: RelationCreatedEvent) -> None: 1191 | """Watch for a relation being created and automatically send dashboards. 1192 | 1193 | Args: 1194 | event: The :class:`RelationJoinedEvent` sent when a 1195 | `grafana_dashboaard` relationship is joined 1196 | """ 1197 | if self._charm.unit.is_leader(): 1198 | self._update_all_dashboards_from_dir() 1199 | self._upset_dashboards_on_relation(event.relation) 1200 | 1201 | def _on_grafana_dashboard_relation_changed(self, event: RelationChangedEvent) -> None: 1202 | """Watch for changes so we know if there's an error to signal back to the parent charm. 1203 | 1204 | Args: 1205 | event: The `RelationChangedEvent` that triggered this handler. 1206 | """ 1207 | if self._charm.unit.is_leader(): 1208 | data = json.loads(event.relation.data[event.app].get("event", "{}")) # type: ignore 1209 | 1210 | if not data: 1211 | return 1212 | 1213 | valid = bool(data.get("valid", True)) 1214 | errors = data.get("errors", []) 1215 | if valid and not errors: 1216 | self.on.dashboard_status_changed.emit(valid=valid) # pyright: ignore 1217 | else: 1218 | self.on.dashboard_status_changed.emit( # pyright: ignore 1219 | valid=valid, errors=errors 1220 | ) 1221 | 1222 | def _upset_dashboards_on_relation(self, relation: Relation) -> None: 1223 | """Update the dashboards in the relation data bucket.""" 1224 | # It's completely ridiculous to add a UUID, but if we don't have some 1225 | # pseudo-random value, this never makes it across 'juju set-state' 1226 | stored_data = { 1227 | "templates": _type_convert_stored(self._stored.dashboard_templates), # pyright: ignore 1228 | "uuid": str(uuid.uuid4()), 1229 | } 1230 | 1231 | relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) 1232 | 1233 | def _content_to_dashboard_object(self, content: str, inject_dropdowns: bool = True) -> Dict: 1234 | return { 1235 | "charm": self._charm.meta.name, 1236 | "content": content, 1237 | "juju_topology": self._juju_topology if inject_dropdowns else {}, 1238 | "inject_dropdowns": inject_dropdowns, 1239 | } 1240 | 1241 | # This is not actually used in the dashboards, but is present to provide a secondary 1242 | # salt to ensure uniqueness in the dict keys in case individual charm units provide 1243 | # dashboards 1244 | @property 1245 | def _juju_topology(self) -> Dict: 1246 | return { 1247 | "model": self._charm.model.name, 1248 | "model_uuid": self._charm.model.uuid, 1249 | "application": self._charm.app.name, 1250 | "unit": self._charm.unit.name, 1251 | } 1252 | 1253 | @property 1254 | def dashboard_templates(self) -> List: 1255 | """Return a list of the known dashboard templates.""" 1256 | return list(self._stored.dashboard_templates.values()) # type: ignore 1257 | 1258 | 1259 | class GrafanaDashboardConsumer(Object): 1260 | """A consumer object for working with Grafana Dashboards.""" 1261 | 1262 | on = GrafanaDashboardEvents() # pyright: ignore 1263 | _stored = StoredState() 1264 | 1265 | def __init__( 1266 | self, 1267 | charm: CharmBase, 1268 | relation_name: str = DEFAULT_RELATION_NAME, 1269 | ) -> None: 1270 | """API to receive Grafana dashboards from charmed operators. 1271 | 1272 | The :class:`GrafanaDashboardConsumer` object provides an API 1273 | to consume dashboards provided by a charmed operator using the 1274 | :class:`GrafanaDashboardProvider` library. The 1275 | :class:`GrafanaDashboardConsumer` is integrated in a 1276 | charmed operator as follows: 1277 | 1278 | self.grafana = GrafanaDashboardConsumer(self) 1279 | 1280 | To use this library, you need a relation defined as follows in 1281 | your charm operator's metadata.yaml: 1282 | 1283 | requires: 1284 | grafana-dashboard: 1285 | interface: grafana_dashboard 1286 | 1287 | If you would like to use a different relation name than 1288 | `grafana-dashboard`, you need to specify the relation name via the 1289 | `relation_name` argument. However, it is strongly advised not to 1290 | change the default, so that people deploying your charm will have 1291 | a consistent experience with all other charms that consume Grafana 1292 | dashboards. 1293 | 1294 | Args: 1295 | charm: a :class:`CharmBase` object which manages this 1296 | :class:`GrafanaProvider` object. Generally this is 1297 | `self` in the instantiating class. 1298 | relation_name: a :string: name of the relation managed by this 1299 | :class:`GrafanaDashboardConsumer`; it defaults to "grafana-dashboard". 1300 | """ 1301 | _validate_relation_by_interface_and_direction( 1302 | charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.requires 1303 | ) 1304 | 1305 | super().__init__(charm, relation_name) 1306 | self._charm = charm 1307 | self._relation_name = relation_name 1308 | self._tranformer = CosTool(self._charm) 1309 | 1310 | self._stored.set_default(dashboards={}) # type: ignore 1311 | 1312 | self.framework.observe( 1313 | self._charm.on[self._relation_name].relation_changed, 1314 | self._on_grafana_dashboard_relation_changed, 1315 | ) 1316 | self.framework.observe( 1317 | self._charm.on[self._relation_name].relation_broken, 1318 | self._on_grafana_dashboard_relation_broken, 1319 | ) 1320 | self.framework.observe( 1321 | self._charm.on[DEFAULT_PEER_NAME].relation_changed, 1322 | self._on_grafana_peer_changed, 1323 | ) 1324 | 1325 | def get_dashboards_from_relation(self, relation_id: int) -> List: 1326 | """Get a list of known dashboards for one instance of the monitored relation. 1327 | 1328 | Args: 1329 | relation_id: the identifier of the relation instance, as returned by 1330 | :method:`ops.model.Relation.id`. 1331 | 1332 | Returns: a list of known dashboards coming from the provided relation instance. 1333 | """ 1334 | return [ 1335 | self._to_external_object(relation_id, dashboard) 1336 | for dashboard in self._get_stored_dashboards(relation_id) 1337 | ] 1338 | 1339 | def _on_grafana_dashboard_relation_changed(self, event: RelationChangedEvent) -> None: 1340 | """Handle relation changes in related providers. 1341 | 1342 | If there are changes in relations between Grafana dashboard consumers 1343 | and providers, this event handler (if the unit is the leader) will 1344 | get data for an incoming grafana-dashboard relation through a 1345 | :class:`GrafanaDashboardsChanged` event, and make the relation data 1346 | available in the app's datastore object. The Grafana charm can 1347 | then respond to the event to update its configuration. 1348 | """ 1349 | changes = False 1350 | if self._charm.unit.is_leader(): 1351 | changes = self._render_dashboards_and_signal_changed(event.relation) 1352 | 1353 | if changes: 1354 | self.on.dashboards_changed.emit() # pyright: ignore 1355 | 1356 | def _on_grafana_peer_changed(self, _: RelationChangedEvent) -> None: 1357 | """Emit dashboard events on peer events so secondary charm data updates.""" 1358 | if self._charm.unit.is_leader(): 1359 | return 1360 | self.on.dashboards_changed.emit() # pyright: ignore 1361 | 1362 | def update_dashboards(self, relation: Optional[Relation] = None) -> None: 1363 | """Re-establish dashboards on one or more relations. 1364 | 1365 | If something changes between this library and a datasource, try to re-establish 1366 | invalid dashboards and invalidate active ones. 1367 | 1368 | Args: 1369 | relation: a specific relation for which the dashboards have to be 1370 | updated. If not specified, all relations managed by this 1371 | :class:`GrafanaDashboardConsumer` will be updated. 1372 | """ 1373 | if self._charm.unit.is_leader(): 1374 | relations = ( 1375 | [relation] if relation else self._charm.model.relations[self._relation_name] 1376 | ) 1377 | 1378 | for relation in relations: 1379 | self._render_dashboards_and_signal_changed(relation) 1380 | 1381 | def _on_grafana_dashboard_relation_broken(self, event: RelationBrokenEvent) -> None: 1382 | """Update job config when providers depart. 1383 | 1384 | When a Grafana dashboard provider departs, the configuration 1385 | for that provider is removed from the list of dashboards 1386 | """ 1387 | if not self._charm.unit.is_leader(): 1388 | return 1389 | 1390 | self._remove_all_dashboards_for_relation(event.relation) 1391 | 1392 | def _render_dashboards_and_signal_changed(self, relation: Relation) -> bool: # type: ignore 1393 | """Validate a given dashboard. 1394 | 1395 | Verify that the passed dashboard data is able to be found in our list 1396 | of datasources and will render. If they do, let the charm know by 1397 | emitting an event. 1398 | 1399 | Args: 1400 | relation: Relation; The relation the dashboard is associated with. 1401 | 1402 | Returns: 1403 | a boolean indicating whether an event should be emitted 1404 | """ 1405 | other_app = relation.app 1406 | 1407 | raw_data = relation.data[other_app].get("dashboards", "") # pyright: ignore 1408 | 1409 | if not raw_data: 1410 | logger.warning( 1411 | "No dashboard data found in the %s:%s relation", 1412 | self._relation_name, 1413 | str(relation.id), 1414 | ) 1415 | return False 1416 | 1417 | data = json.loads(raw_data) 1418 | 1419 | # The only piece of data needed on this side of the relations is "templates" 1420 | templates = data.pop("templates") 1421 | 1422 | # The dashboards are WAY too big since this ultimately calls out to Juju to 1423 | # set the relation data, and it overflows the maximum argument length for 1424 | # subprocess, so we have to use b64, annoyingly. 1425 | # Worse, Python3 expects absolutely everything to be a byte, and a plain 1426 | # `base64.b64encode()` is still too large, so we have to go through hoops 1427 | # of encoding to byte, compressing with lzma, converting to base64 so it 1428 | # can be converted to JSON, then all the way back. 1429 | 1430 | rendered_dashboards = [] 1431 | relation_has_invalid_dashboards = False 1432 | 1433 | for _, (fname, template) in enumerate(templates.items()): 1434 | content = None 1435 | error = None 1436 | topology = template.get("juju_topology", {}) 1437 | try: 1438 | content = _decode_dashboard_content(template["content"]) 1439 | inject_dropdowns = template.get("inject_dropdowns", True) 1440 | content = self._manage_dashboard_uid(content, template) 1441 | content = _convert_dashboard_fields(content, inject_dropdowns) 1442 | 1443 | if topology: 1444 | content = _inject_labels(content, topology, self._tranformer) 1445 | 1446 | content = _encode_dashboard_content(content) 1447 | except lzma.LZMAError as e: 1448 | error = str(e) 1449 | relation_has_invalid_dashboards = True 1450 | except json.JSONDecodeError as e: 1451 | error = str(e.msg) 1452 | logger.warning("Invalid JSON in Grafana dashboard: {}".format(fname)) 1453 | continue 1454 | 1455 | # Prepend the relation name and ID to the dashboard ID to avoid clashes with 1456 | # multiple relations with apps from the same charm, or having dashboards with 1457 | # the same ids inside their charm operators 1458 | rendered_dashboards.append( 1459 | { 1460 | "id": "{}:{}/{}".format(relation.name, relation.id, fname), 1461 | "original_id": fname, 1462 | "content": content if content else None, 1463 | "template": template, 1464 | "valid": (error is None), 1465 | "error": error, 1466 | } 1467 | ) 1468 | 1469 | if relation_has_invalid_dashboards: 1470 | self._remove_all_dashboards_for_relation(relation) 1471 | 1472 | invalid_templates = [ 1473 | data["original_id"] for data in rendered_dashboards if not data["valid"] 1474 | ] 1475 | 1476 | logger.warning( 1477 | "Cannot add one or more Grafana dashboards from relation '{}:{}': the following " 1478 | "templates are invalid: {}".format( 1479 | relation.name, 1480 | relation.id, 1481 | invalid_templates, 1482 | ) 1483 | ) 1484 | 1485 | relation.data[self._charm.app]["event"] = json.dumps( 1486 | { 1487 | "errors": [ 1488 | { 1489 | "dashboard_id": rendered_dashboard["original_id"], 1490 | "error": rendered_dashboard["error"], 1491 | } 1492 | for rendered_dashboard in rendered_dashboards 1493 | if rendered_dashboard["error"] 1494 | ] 1495 | } 1496 | ) 1497 | 1498 | # Dropping dashboards for a relation needs to be signalled 1499 | return True 1500 | 1501 | stored_data = rendered_dashboards 1502 | currently_stored_data = self._get_stored_dashboards(relation.id) 1503 | 1504 | coerced_data = _type_convert_stored(currently_stored_data) if currently_stored_data else {} 1505 | 1506 | if not coerced_data == stored_data: 1507 | stored_dashboards = self.get_peer_data("dashboards") 1508 | stored_dashboards[relation.id] = stored_data 1509 | self.set_peer_data("dashboards", stored_dashboards) 1510 | return True 1511 | return None # type: ignore 1512 | 1513 | def _manage_dashboard_uid(self, dashboard: str, template: dict) -> str: 1514 | """Add an uid to the dashboard if it is not present.""" 1515 | dashboard_dict = json.loads(dashboard) 1516 | 1517 | if not dashboard_dict.get("uid", None) and "dashboard_alt_uid" in template: 1518 | dashboard_dict["uid"] = template["dashboard_alt_uid"] 1519 | 1520 | return json.dumps(dashboard_dict) 1521 | 1522 | def _remove_all_dashboards_for_relation(self, relation: Relation) -> None: 1523 | """If an errored dashboard is in stored data, remove it and trigger a deletion.""" 1524 | if self._get_stored_dashboards(relation.id): 1525 | stored_dashboards = self.get_peer_data("dashboards") 1526 | stored_dashboards.pop(str(relation.id)) 1527 | self.set_peer_data("dashboards", stored_dashboards) 1528 | self.on.dashboards_changed.emit() # pyright: ignore 1529 | 1530 | def _to_external_object(self, relation_id, dashboard): 1531 | return { 1532 | "id": dashboard["original_id"], 1533 | "relation_id": relation_id, 1534 | "charm": dashboard["template"]["charm"], 1535 | "content": _decode_dashboard_content(dashboard["content"]), 1536 | } 1537 | 1538 | @property 1539 | def dashboards(self) -> List[Dict]: 1540 | """Get a list of known dashboards across all instances of the monitored relation. 1541 | 1542 | Returns: a list of known dashboards. The JSON of each of the dashboards is available 1543 | in the `content` field of the corresponding `dict`. 1544 | """ 1545 | dashboards = [] 1546 | 1547 | for _, (relation_id, dashboards_for_relation) in enumerate( 1548 | self.get_peer_data("dashboards").items() 1549 | ): 1550 | for dashboard in dashboards_for_relation: 1551 | dashboards.append(self._to_external_object(relation_id, dashboard)) 1552 | 1553 | return dashboards 1554 | 1555 | def _get_stored_dashboards(self, relation_id: int) -> list: 1556 | """Pull stored dashboards out of the peer data bucket.""" 1557 | return self.get_peer_data("dashboards").get(str(relation_id), {}) 1558 | 1559 | def _set_default_data(self) -> None: 1560 | """Set defaults if they are not in peer relation data.""" 1561 | data = {"dashboards": {}} # type: ignore 1562 | for k, v in data.items(): 1563 | if not self.get_peer_data(k): 1564 | self.set_peer_data(k, v) 1565 | 1566 | def set_peer_data(self, key: str, data: Any) -> None: 1567 | """Put information into the peer data bucket instead of `StoredState`.""" 1568 | self._charm.peers.data[self._charm.app][key] = json.dumps(data) # type: ignore[attr-defined] 1569 | 1570 | def get_peer_data(self, key: str) -> Any: 1571 | """Retrieve information from the peer data bucket instead of `StoredState`.""" 1572 | data = self._charm.peers.data[self._charm.app].get(key, "") # type: ignore[attr-defined] 1573 | return json.loads(data) if data else {} 1574 | 1575 | 1576 | class GrafanaDashboardAggregator(Object): 1577 | """API to retrieve Grafana dashboards from machine dashboards. 1578 | 1579 | The :class:`GrafanaDashboardAggregator` object provides a way to 1580 | collate and aggregate Grafana dashboards from reactive/machine charms 1581 | and transport them into Charmed Operators, using Juju topology. 1582 | For detailed usage instructions, see the documentation for 1583 | :module:`cos-proxy-operator`, as this class is intended for use as a 1584 | single point of intersection rather than use in individual charms. 1585 | 1586 | Since :class:`GrafanaDashboardAggregator` serves as a bridge between 1587 | Canonical Observability Stack Charmed Operators and Reactive Charms, 1588 | deployed in a Reactive Juju model, both a target relation which is 1589 | used to collect events from Reactive charms and a `grafana_relation` 1590 | which is used to send the collected data back to the Canonical 1591 | Observability Stack are required. 1592 | 1593 | In its most streamlined usage, :class:`GrafanaDashboardAggregator` is 1594 | integrated in a charmed operator as follows: 1595 | self.grafana = GrafanaDashboardAggregator(self) 1596 | 1597 | Args: 1598 | charm: a :class:`CharmBase` object which manages this 1599 | :class:`GrafanaProvider` object. Generally this is 1600 | `self` in the instantiating class. 1601 | target_relation: a :string: name of a relation managed by this 1602 | :class:`GrafanaDashboardAggregator`, which is used to communicate 1603 | with reactive/machine charms it defaults to "dashboards". 1604 | grafana_relation: a :string: name of a relation used by this 1605 | :class:`GrafanaDashboardAggregator`, which is used to communicate 1606 | with charmed grafana. It defaults to "downstream-grafana-dashboard" 1607 | """ 1608 | 1609 | _stored = StoredState() 1610 | on = GrafanaProviderEvents() # pyright: ignore 1611 | 1612 | def __init__( 1613 | self, 1614 | charm: CharmBase, 1615 | target_relation: str = "dashboards", 1616 | grafana_relation: str = "downstream-grafana-dashboard", 1617 | ): 1618 | super().__init__(charm, grafana_relation) 1619 | 1620 | # Reactive charms may be RPC-ish and not leave reliable data around. Keep 1621 | # StoredState here 1622 | self._stored.set_default( # type: ignore 1623 | dashboard_templates={}, 1624 | id_mappings={}, 1625 | ) 1626 | 1627 | self._charm = charm 1628 | self._target_relation = target_relation 1629 | self._grafana_relation = grafana_relation 1630 | 1631 | self.framework.observe( 1632 | self._charm.on[self._grafana_relation].relation_joined, 1633 | self._update_remote_grafana, 1634 | ) 1635 | self.framework.observe( 1636 | self._charm.on[self._grafana_relation].relation_changed, 1637 | self._update_remote_grafana, 1638 | ) 1639 | self.framework.observe( 1640 | self._charm.on[self._target_relation].relation_changed, 1641 | self.update_dashboards, 1642 | ) 1643 | self.framework.observe( 1644 | self._charm.on[self._target_relation].relation_broken, 1645 | self.remove_dashboards, 1646 | ) 1647 | 1648 | def update_dashboards(self, event: RelationEvent) -> None: 1649 | """If we get a dashboard from a reactive charm, parse it out and update.""" 1650 | if self._charm.unit.is_leader(): 1651 | self._upset_dashboards_on_event(event) 1652 | 1653 | def _upset_dashboards_on_event(self, event: RelationEvent) -> None: 1654 | """Update the dashboards in the relation data bucket.""" 1655 | dashboards = self._handle_reactive_dashboards(event) 1656 | 1657 | if not dashboards: 1658 | logger.warning( 1659 | "Could not find dashboard data after a relation change for {}".format(event.app) 1660 | ) 1661 | return 1662 | 1663 | for id in dashboards: 1664 | self._stored.dashboard_templates[id] = self._content_to_dashboard_object( # type: ignore 1665 | dashboards[id], event 1666 | ) 1667 | 1668 | self._stored.id_mappings[event.app.name] = dashboards # type: ignore 1669 | self._update_remote_grafana(event) 1670 | 1671 | def _update_remote_grafana(self, _: Optional[RelationEvent] = None) -> None: 1672 | """Push dashboards to the downstream Grafana relation.""" 1673 | # It's still ridiculous to add a UUID here, but needed 1674 | stored_data = { 1675 | "templates": _type_convert_stored(self._stored.dashboard_templates), # pyright: ignore 1676 | "uuid": str(uuid.uuid4()), 1677 | } 1678 | 1679 | if self._charm.unit.is_leader(): 1680 | for grafana_relation in self.model.relations[self._grafana_relation]: 1681 | grafana_relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) 1682 | 1683 | def remove_dashboards(self, event: RelationBrokenEvent) -> None: 1684 | """Remove a dashboard if the relation is broken.""" 1685 | app_ids = _type_convert_stored(self._stored.id_mappings.get(event.app.name, "")) # type: ignore 1686 | 1687 | if not app_ids: 1688 | logger.info("Could not look up stored dashboards for %s", event.app.name) # type: ignore 1689 | return 1690 | 1691 | del self._stored.id_mappings[event.app.name] # type: ignore 1692 | for id in app_ids: 1693 | del self._stored.dashboard_templates[id] # type: ignore 1694 | 1695 | stored_data = { 1696 | "templates": _type_convert_stored(self._stored.dashboard_templates), # pyright: ignore 1697 | "uuid": str(uuid.uuid4()), 1698 | } 1699 | 1700 | if self._charm.unit.is_leader(): 1701 | for grafana_relation in self.model.relations[self._grafana_relation]: 1702 | grafana_relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) 1703 | 1704 | # Yes, this has a fair amount of branching. It's not that complex, though 1705 | def _strip_existing_datasources(self, dash: dict) -> dict: # noqa: C901 1706 | """Remove existing reactive charm datasource templating out. 1707 | 1708 | This method iterates through *known* places where reactive charms may set 1709 | data in contributed dashboards and removes them. 1710 | 1711 | `dashboard["__inputs"]` is a property sometimes set when exporting dashboards from 1712 | the Grafana UI. It is not present in earlier Grafana versions, and can be disabled 1713 | in 5.3.4 and above (optionally). If set, any values present will be substituted on 1714 | import. Some reactive charms use this for Prometheus. COS uses dropdown selectors 1715 | for datasources, and leaving this present results in "default" datasource values 1716 | which are broken. 1717 | 1718 | Similarly, `dashboard["templating"]["list"][N]["name"] == "host"` can be used to 1719 | set a `host` variable for use in dashboards which is not meaningful in the context 1720 | of Juju topology and will yield broken dashboards. 1721 | 1722 | Further properties may be discovered. 1723 | """ 1724 | try: 1725 | if "list" in dash["templating"]: 1726 | for i in range(len(dash["templating"]["list"])): 1727 | if ( 1728 | "datasource" in dash["templating"]["list"][i] 1729 | and dash["templating"]["list"][i]["datasource"] is not None 1730 | ): 1731 | if "Juju" in dash["templating"]["list"][i].get("datasource", ""): 1732 | dash["templating"]["list"][i]["datasource"] = r"${prometheusds}" 1733 | 1734 | # Strip out newly-added 'juju_application' template variables which 1735 | # don't line up with our drop-downs 1736 | dash_mutable = dash 1737 | for i in range(len(dash["templating"]["list"])): 1738 | if ( 1739 | "name" in dash["templating"]["list"][i] 1740 | and dash["templating"]["list"][i].get("name", "") == "app" 1741 | ): 1742 | del dash_mutable["templating"]["list"][i] 1743 | 1744 | if dash_mutable: 1745 | dash = dash_mutable 1746 | except KeyError: 1747 | logger.debug("No existing templating data in dashboard") 1748 | 1749 | if "__inputs" in dash: 1750 | inputs = dash 1751 | for i in range(len(dash["__inputs"])): 1752 | if dash["__inputs"][i].get("pluginName", "") == "Prometheus": 1753 | del inputs["__inputs"][i] 1754 | if inputs: 1755 | dash["__inputs"] = inputs["__inputs"] 1756 | else: 1757 | del dash["__inputs"] 1758 | 1759 | return dash 1760 | 1761 | def _handle_reactive_dashboards(self, event: RelationEvent) -> Optional[Dict]: 1762 | """Look for a dashboard in relation data (during a reactive hook) or builtin by name.""" 1763 | if not self._charm.unit.is_leader(): 1764 | return {} 1765 | 1766 | templates = [] 1767 | id = "" 1768 | 1769 | # Reactive data can reliably be pulled out of events. In theory, if we got an event, 1770 | # it's on the bucket, but using event explicitly keeps the mental model in 1771 | # place for reactive 1772 | for k in event.relation.data[event.unit].keys(): # type: ignore 1773 | if k.startswith("request_"): 1774 | templates.append(json.loads(event.relation.data[event.unit][k])["dashboard"]) # type: ignore 1775 | 1776 | for k in event.relation.data[event.app].keys(): # type: ignore 1777 | if k.startswith("request_"): 1778 | templates.append(json.loads(event.relation.data[event.app][k])["dashboard"]) # type: ignore 1779 | 1780 | builtins = self._maybe_get_builtin_dashboards(event) 1781 | 1782 | if not templates and not builtins: 1783 | logger.warning("NOTHING!") 1784 | return {} 1785 | 1786 | dashboards = {} 1787 | for t in templates: 1788 | # This seems ridiculous, too, but to get it from a "dashboards" key in serialized JSON 1789 | # in the bucket back out to the actual "dashboard" we _need_, this is the way 1790 | # This is not a mistake -- there's a double nesting in reactive charms, and 1791 | # Grafana won't load it. We have to unbox: 1792 | # event.relation.data[event.]["request_*"]["dashboard"]["dashboard"], 1793 | # and the final unboxing is below. 1794 | # 1795 | # Apparently SOME newer dashboards (such as Ceph) do not have this double nesting, so 1796 | # now we get to account for both :toot: 1797 | dash = t.get("dashboard", {}) or t 1798 | 1799 | # Replace values with LMA-style templating 1800 | dash = self._strip_existing_datasources(dash) 1801 | dash = json.dumps(dash) 1802 | 1803 | # Replace the old-style datasource templates 1804 | dash = re.sub(r"<< datasource >>", r"${prometheusds}", dash) 1805 | dash = re.sub(r'"datasource": "prom.*?"', r'"datasource": "${prometheusds}"', dash) 1806 | dash = re.sub( 1807 | r'"datasource": "\$datasource"', r'"datasource": "${prometheusds}"', dash 1808 | ) 1809 | dash = re.sub(r'"uid": "\$datasource"', r'"uid": "${prometheusds}"', dash) 1810 | dash = re.sub( 1811 | r'"datasource": "(!?\w)[\w|\s|-]+?Juju generated.*?"', 1812 | r'"datasource": "${prometheusds}"', 1813 | dash, 1814 | ) 1815 | 1816 | # Yank out "new"+old LMA topology 1817 | dash = re.sub( 1818 | r'(,?\s?juju_application=~)\\"\$app\\"', r'\1\\"$juju_application\\"', dash 1819 | ) 1820 | 1821 | # Replace old piechart panels 1822 | dash = re.sub(r'"type": "grafana-piechart-panel"', '"type": "piechart"', dash) 1823 | 1824 | from jinja2 import DebugUndefined, Template 1825 | 1826 | content = _encode_dashboard_content( 1827 | Template(dash, undefined=DebugUndefined).render(datasource=r"${prometheusds}") # type: ignore 1828 | ) 1829 | id = "prog:{}".format(content[-24:-16]) 1830 | 1831 | dashboards[id] = content 1832 | return {**builtins, **dashboards} 1833 | 1834 | def _maybe_get_builtin_dashboards(self, event: RelationEvent) -> Dict: 1835 | """Tries to match the event with an included dashboard. 1836 | 1837 | Scans dashboards packed with the charm instantiating this class, and tries to match 1838 | one with the event. There is no guarantee that any given event will match a builtin, 1839 | since each charm instantiating this class may include a different set of dashboards, 1840 | or none. 1841 | """ 1842 | builtins = {} 1843 | dashboards_path = None 1844 | 1845 | try: 1846 | dashboards_path = _resolve_dir_against_charm_path( 1847 | self._charm, "src/grafana_dashboards" 1848 | ) 1849 | except InvalidDirectoryPathError as e: 1850 | logger.warning( 1851 | "Invalid Grafana dashboards folder at %s: %s", 1852 | e.grafana_dashboards_absolute_path, 1853 | e.message, 1854 | ) 1855 | 1856 | if dashboards_path: 1857 | 1858 | def is_dashboard(p: Path) -> bool: 1859 | return p.is_file() and p.name.endswith((".json", ".json.tmpl", ".tmpl")) 1860 | 1861 | for path in filter(is_dashboard, Path(dashboards_path).glob("*")): 1862 | # path = Path(path) 1863 | if event.app.name in path.name: # type: ignore 1864 | id = "file:{}".format(path.stem) 1865 | builtins[id] = self._content_to_dashboard_object( 1866 | _encode_dashboard_content(path.read_bytes()), event 1867 | ) 1868 | 1869 | return builtins 1870 | 1871 | def _content_to_dashboard_object(self, content: str, event: RelationEvent) -> Dict: 1872 | return { 1873 | "charm": event.app.name, # type: ignore 1874 | "content": content, 1875 | "juju_topology": self._juju_topology(event), 1876 | "inject_dropdowns": True, 1877 | } 1878 | 1879 | # This is not actually used in the dashboards, but is present to provide a secondary 1880 | # salt to ensure uniqueness in the dict keys in case individual charm units provide 1881 | # dashboards 1882 | def _juju_topology(self, event: RelationEvent) -> Dict: 1883 | return { 1884 | "model": self._charm.model.name, 1885 | "model_uuid": self._charm.model.uuid, 1886 | "application": event.app.name, # type: ignore 1887 | "unit": event.unit.name, # type: ignore 1888 | } 1889 | 1890 | 1891 | class CosTool: 1892 | """Uses cos-tool to inject label matchers into alert rule expressions and validate rules.""" 1893 | 1894 | _path = None 1895 | _disabled = False 1896 | 1897 | def __init__(self, charm): 1898 | self._charm = charm 1899 | 1900 | @property 1901 | def path(self): 1902 | """Lazy lookup of the path of cos-tool.""" 1903 | if self._disabled: 1904 | return None 1905 | if not self._path: 1906 | self._path = self._get_tool_path() 1907 | if not self._path: 1908 | logger.debug("Skipping injection of juju topology as label matchers") 1909 | self._disabled = True 1910 | return self._path 1911 | 1912 | def apply_label_matchers(self, rules: dict, type: str) -> dict: 1913 | """Will apply label matchers to the expression of all alerts in all supplied groups.""" 1914 | if not self.path: 1915 | return rules 1916 | for group in rules["groups"]: 1917 | rules_in_group = group.get("rules", []) 1918 | for rule in rules_in_group: 1919 | topology = {} 1920 | # if the user for some reason has provided juju_unit, we'll need to honor it 1921 | # in most cases, however, this will be empty 1922 | for label in [ 1923 | "juju_model", 1924 | "juju_model_uuid", 1925 | "juju_application", 1926 | "juju_charm", 1927 | "juju_unit", 1928 | ]: 1929 | if label in rule["labels"]: 1930 | topology[label] = rule["labels"][label] 1931 | 1932 | rule["expr"] = self.inject_label_matchers(rule["expr"], topology, type) 1933 | return rules 1934 | 1935 | def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: 1936 | """Will validate correctness of alert rules, returning a boolean and any errors.""" 1937 | if not self.path: 1938 | logger.debug("`cos-tool` unavailable. Not validating alert correctness.") 1939 | return True, "" 1940 | 1941 | with tempfile.TemporaryDirectory() as tmpdir: 1942 | rule_path = Path(tmpdir + "/validate_rule.yaml") 1943 | 1944 | # Smash "our" rules format into what upstream actually uses, which is more like: 1945 | # 1946 | # groups: 1947 | # - name: foo 1948 | # rules: 1949 | # - alert: SomeAlert 1950 | # expr: up 1951 | # - alert: OtherAlert 1952 | # expr: up 1953 | transformed_rules = {"groups": []} # type: ignore 1954 | for rule in rules["groups"]: 1955 | transformed = {"name": str(uuid.uuid4()), "rules": [rule]} 1956 | transformed_rules["groups"].append(transformed) 1957 | 1958 | rule_path.write_text(yaml.dump(transformed_rules)) 1959 | 1960 | args = [str(self.path), "validate", str(rule_path)] 1961 | # noinspection PyBroadException 1962 | try: 1963 | self._exec(args) 1964 | return True, "" 1965 | except subprocess.CalledProcessError as e: 1966 | logger.debug("Validating the rules failed: %s", e.output) 1967 | return False, ", ".join([line for line in e.output if "error validating" in line]) 1968 | 1969 | def inject_label_matchers(self, expression: str, topology: dict, type: str) -> str: 1970 | """Add label matchers to an expression.""" 1971 | if not topology: 1972 | return expression 1973 | if not self.path: 1974 | logger.debug("`cos-tool` unavailable. Leaving expression unchanged: %s", expression) 1975 | return expression 1976 | args = [str(self.path), "--format", type, "transform"] 1977 | 1978 | variable_topology = {k: "${}".format(k) for k in topology.keys()} 1979 | args.extend( 1980 | [ 1981 | "--label-matcher={}={}".format(key, value) 1982 | for key, value in variable_topology.items() 1983 | ] 1984 | ) 1985 | 1986 | # Pass a leading "--" so expressions with a negation or subtraction aren't interpreted as 1987 | # flags 1988 | args.extend(["--", "{}".format(expression)]) 1989 | # noinspection PyBroadException 1990 | try: 1991 | return re.sub(r'="\$juju', r'=~"$juju', self._exec(args)) 1992 | except subprocess.CalledProcessError as e: 1993 | logger.debug('Applying the expression failed: "%s", falling back to the original', e) 1994 | return expression 1995 | 1996 | def _get_tool_path(self) -> Optional[Path]: 1997 | arch = platform.machine() 1998 | arch = "amd64" if arch == "x86_64" else arch 1999 | res = "cos-tool-{}".format(arch) 2000 | try: 2001 | path = Path(res).resolve() 2002 | path.chmod(0o777) 2003 | return path 2004 | except NotImplementedError: 2005 | logger.debug("System lacks support for chmod") 2006 | except FileNotFoundError: 2007 | logger.debug('Could not locate cos-tool at: "{}"'.format(res)) 2008 | return None 2009 | 2010 | def _exec(self, cmd) -> str: 2011 | result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE) 2012 | output = result.stdout.decode("utf-8").strip() 2013 | return output 2014 | --------------------------------------------------------------------------------