├── automated-networkpolicy-generation ├── 1-inject-sidecar.sh ├── 2-copy-capture-and-metadata.sh ├── 3-analyse.py ├── 4-delete-sidecar.sh ├── README.md ├── create-capture-metadata.py ├── env.sh ├── graph_ms_demo.png ├── graph_simple.png ├── requirements.txt ├── system-overall.drawio ├── system-overall.png └── tcpdump-sidecar-patch.yaml ├── debug-k8s-workload-1 ├── Dockerfile ├── diagram-k8s-debug.bpmn ├── skaffold.yaml └── viewer │ ├── index.html │ └── resources │ ├── diagram-k8s-debug.bpmn │ └── screenshot.png ├── network-policies ├── README.md ├── cilium-editor.png ├── network_perimeter_kubernetes.drawio └── network_perimeter_kubernetes.png ├── opa-policies └── conformance-testing │ ├── README.MD │ ├── dashboard-screenshot.png │ ├── helpers │ ├── convert_to_python_map.py │ ├── deployment.yaml │ └── requirements.txt │ └── policy │ ├── deployment.rego │ └── deployment_test.rego ├── prometheus-advanced ├── README.MD ├── inventory-local.yaml ├── nginx-ingress │ └── values.yml ├── playbooks │ ├── nginx-ingress.yml │ └── prometheus-operator.yml ├── prometheus-operator │ └── values.yml.j2 ├── system-overall.drawio ├── system-overall.png └── vars │ └── test.yml ├── tmc-data-protection ├── README.md ├── account_creation_drop_down.png ├── admin_accounts.png ├── cluster.png ├── clusters.png ├── create_aws_dp_provider_credential.png ├── create_aws_dp_provider_credential_step2.png ├── create_aws_dp_provider_credential_step3.png ├── dp-account.png └── enable-data-protection.png └── velero-1 ├── README.md ├── creds.txt ├── example-app-with-pv.yaml ├── values.yaml └── velero.png /automated-networkpolicy-generation/1-inject-sidecar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source env.sh 4 | DEPLOYMENTS=$(kubectl get deployment -n $TARGET_NS -o name) 5 | 6 | for d in $DEPLOYMENTS 7 | do 8 | kubectl -n $TARGET_NS patch $d --patch "$(cat tcpdump-sidecar-patch.yaml)" 9 | done 10 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/2-copy-capture-and-metadata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source env.sh 4 | 5 | set -eu 6 | 7 | PODS=$(kubectl get pods -n $TARGET_NS -o name --field-selector status.phase!=Terminating | cut -d/ -f2) 8 | 9 | i=0 10 | 11 | until [ $i -gt $TEST_DURATION_IN_SECONDS ] 12 | do 13 | echo "Going to wait for $((TEST_DURATION_IN_SECONDS-i)) seconds so that application traffic can be generated..." 14 | ((i=i+1)) 15 | sleep 1 16 | done 17 | 18 | mkdir -p .tmp 19 | for p in $PODS 20 | do 21 | FILE="${p}.pcap" 22 | STATUS=$(kubectl get pod ${p} -o jsonpath='{.status.phase}' -n $TARGET_NS --ignore-not-found) 23 | if [[ $STATUS != "Running" ]] ; 24 | then 25 | echo "Pod ${p} is not in 'Running' state" 26 | else 27 | kubectl -n $TARGET_NS cp $p:/tmp/tcpdump.pcap ".tmp/${FILE}" -c tcpdump 28 | fi 29 | 30 | done 31 | 32 | ./create-capture-metadata.py $TARGET_NS ${PODS} 33 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/3-analyse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import subprocess 4 | import logging 5 | import json 6 | import os 7 | import shutil 8 | 9 | from enum import Enum 10 | from dataclasses import dataclass 11 | from typing import Sequence 12 | 13 | from graph_tools import Graph 14 | from kubernetes import client 15 | from kubernetes.client import V1IPBlock 16 | 17 | map_ip_2_svc = {} 18 | map_name_2_pod = {} 19 | map_name_2_rs = {} 20 | map_name_2_deployments = {} 21 | 22 | g = Graph() 23 | logger = logging.getLogger('traffic_analyser') 24 | 25 | EDGE_ATTRIBUTE_PORT_ID = 0 26 | EDGE_ATTRIBUTE_PORT_NAME = 'ports' 27 | VERTEX_ATTRIBUTE_POD = 'pod' 28 | 29 | include_ingress_in_policy = True 30 | include_egress_in_policy = True 31 | create_dns_policy = True 32 | 33 | 34 | def process_file(file): 35 | base_folder = file[: file.rindex('/')] 36 | with open(file, "r") as a_file: 37 | capture_metadata = json.loads(a_file.read()) 38 | build_service_map(capture_metadata["services"]["items"]) 39 | build_pod_map(capture_metadata["pods"]["items"]) 40 | build_rs_map(capture_metadata["rs"]["items"]) 41 | build_deployments_map(capture_metadata["deployments"]["items"]) 42 | for pod_metadata in capture_metadata["pod_metadata"]: 43 | handle_capture_per_pod(pod_metadata["pod"], base_folder + '/' + pod_metadata["file"], 44 | pod_metadata["IP"]) 45 | 46 | print_graph() 47 | result = create_network_policy() 48 | policy_folder = base_folder + '/network-policies' 49 | create_clean_policies_directory(policy_folder) 50 | for policy in result: 51 | policy_content = client.ApiClient().sanitize_for_serialization(policy) 52 | with open(policy_folder + '/network_policy_{}.json'.format(policy_content["metadata"]["name"]), "w") as policy_file: 53 | policy_file.write(json.dumps(policy_content, indent=4)) 54 | 55 | 56 | def create_clean_policies_directory(dir): 57 | try: 58 | shutil.rmtree(dir, ignore_errors=True, onerror=None) 59 | except: 60 | logger.warning('Error removing dir:{}'.format(dir)) 61 | os.mkdir(dir) 62 | 63 | 64 | def build_service_map(service_items): 65 | for svc in service_items: 66 | map_ip_2_svc[svc["spec"]["clusterIP"]] = svc 67 | 68 | 69 | def build_pod_map(pod_items): 70 | for pod in pod_items: 71 | map_name_2_pod[pod["metadata"]["name"]] = pod 72 | 73 | 74 | def build_rs_map(rs_items): 75 | for rs in rs_items: 76 | map_name_2_rs[rs["metadata"]["name"]] = rs 77 | 78 | 79 | def build_deployments_map(deployment_items): 80 | for d in deployment_items: 81 | map_name_2_deployments[d["metadata"]["name"]] = d 82 | 83 | 84 | def handle_capture_per_pod(pod_name, capture_file_path, pod_ip): 85 | logger.debug('Pod: {}, Capture: {}, IP: {}'.format(pod_name, capture_file_path, pod_ip)) 86 | result = subprocess.check_output( 87 | """ tshark -r {} -Y " ip.src == {} && tcp.flags.syn==1 && tcp.flags.ack==0 && not icmp" -Tfields \ 88 | -eip.dst_host -e tcp.dstport -Eseparator=, | sort --unique """.format( 89 | capture_file_path, pod_ip), shell=True, timeout=60, stderr=None).decode("utf-8") 90 | process_stdout_from_tshark(pod_name, result) 91 | 92 | 93 | def process_stdout_from_tshark(pod_name, result): 94 | if not result or len(result) == 0: 95 | logger.info("Empty response!") 96 | return 97 | 98 | target_hosts = result.split("\n") 99 | for target_host in target_hosts: 100 | if len(target_host) == 0: 101 | continue 102 | parts = target_host.split(",") 103 | target_ip = parts[0] 104 | target_port = parts[1] 105 | logger.debug('IP:{}, Port:{}'.format(target_ip, target_port)) 106 | push_edge_information(pod_name, target_ip, target_port) 107 | 108 | 109 | @dataclass(frozen=True, eq=True, order=True) 110 | class NodeType(Enum): 111 | POD = 1 112 | IP = 2 113 | FQDN = 3 114 | 115 | 116 | @dataclass(frozen=True, eq=True, order=True) 117 | class Node: 118 | type: NodeType 119 | name: str 120 | id: str 121 | 122 | def __repr__(self): 123 | return '<%s: %s>' % ( 124 | self.type.name, self.name) 125 | 126 | 127 | def push_edge_information(source_pod_name, target_ip, target_port): 128 | logger.info('Source Pod:{}, Destination IP:{}, Destination Port:{}'.format(source_pod_name, target_ip, target_port)) 129 | if target_ip.startswith('169.254'): # Ignore link-local address, https://en.wikipedia.org/wiki/Link-local_address 130 | logger.info('Skipping ip: {}'.format(target_ip)) 131 | return 132 | 133 | target = None 134 | target_pod_name = None 135 | source = None 136 | 137 | if map_ip_2_svc.keys().__contains__(target_ip): 138 | target_pod_name = find_pods_that_resolves_to_svc(map_ip_2_svc[target_ip]) 139 | else: 140 | logger.info('No service found for IP:{}'.format(target_ip)) 141 | target_pod_name = find_pod_with_ip(target_ip) 142 | 143 | if target_pod_name is None or target_pod_name == '': 144 | logger.warning('No target svc or pod found for IP: {}, assuming that this is an external IP.'.format(target_ip)) 145 | target = Node(NodeType.IP, target_ip , target_ip) 146 | else: 147 | target = Node(NodeType.POD, look_up_owner_name(target_pod_name), target_pod_name) 148 | 149 | source = Node(NodeType.POD, look_up_owner_name(source_pod_name), source_pod_name) 150 | g.add_vertex(source) 151 | 152 | 153 | g.add_vertex(target) 154 | g.add_edge(source, target) 155 | # Add port to edge properties 156 | existing = None 157 | try: 158 | existing = g.get_edge_attribute_by_id(source, target, EDGE_ATTRIBUTE_PORT_ID, EDGE_ATTRIBUTE_PORT_NAME) 159 | except: 160 | logger.warning("Edge attribute not found, source:{}, target: {}", source, target) 161 | if not existing: 162 | existing = [] 163 | existing.append(target_port) 164 | g.set_edge_attribute_by_id(source, target, EDGE_ATTRIBUTE_PORT_ID, EDGE_ATTRIBUTE_PORT_NAME, existing) 165 | 166 | 167 | def find_pod_with_ip(ip): 168 | for pod in map_name_2_pod.values(): 169 | if pod["status"]["podIP"] == ip: 170 | return pod["metadata"]["name"] 171 | return '' 172 | 173 | 174 | def create_network_policy(): 175 | result = [] 176 | for v in g.vertices(): 177 | # Only handle pods, other types should be handled as dependencies with their relation to pods 178 | if v.type is not NodeType.POD: 179 | continue 180 | 181 | network_policy: client.models.V1NetworkPolicy = None 182 | if include_egress_in_policy: 183 | for egress in g.edges_from(v): 184 | logger.debug("Egress from edge: {}", egress) 185 | ports = g.get_edge_attribute_by_id(egress[0], egress[1], EDGE_ATTRIBUTE_PORT_ID, EDGE_ATTRIBUTE_PORT_NAME) 186 | if network_policy is None: 187 | network_policy = create_network_policy_skeleton(v) 188 | if network_policy.spec.egress is None: 189 | network_policy.spec.egress = [] 190 | network_policy.spec.egress.append( client.models.V1NetworkPolicyEgressRule( 191 | ports=create_network_policy_ports(ports))) 192 | if network_policy.spec.egress[len(network_policy.spec.egress) -1]._to is None: 193 | network_policy.spec.egress[len(network_policy.spec.egress) -1]._to = [] 194 | network_policy.spec.egress[len(network_policy.spec.egress) -1]._to.append(create_network_policy_peer(egress[1])) 195 | 196 | if include_ingress_in_policy: 197 | for ingress in g.edges_to(v): 198 | logger.debug("Ingress to edge: {}", ingress) 199 | ports = g.get_edge_attribute_by_id(ingress[0], ingress[1], EDGE_ATTRIBUTE_PORT_ID, EDGE_ATTRIBUTE_PORT_NAME) 200 | if network_policy is None: 201 | network_policy = create_network_policy_skeleton(v) 202 | if network_policy.spec.ingress is None: 203 | network_policy.spec.ingress = [] 204 | network_policy.spec.ingress.append(client.models.V1NetworkPolicyIngressRule( 205 | ports=create_network_policy_ports(ports))) 206 | if network_policy.spec.ingress[len(network_policy.spec.ingress) -1]._from is None: 207 | network_policy.spec.ingress[len(network_policy.spec.ingress) -1]._from = [] 208 | network_policy.spec.ingress[len(network_policy.spec.ingress) -1]._from.append(create_network_policy_peer(ingress[0])) 209 | 210 | update_policy_types(network_policy) 211 | result.append(network_policy) 212 | 213 | if create_dns_policy: 214 | result.append(create_dns_network_policy()) 215 | 216 | return result 217 | 218 | 219 | def create_dns_network_policy(): 220 | np = client.models.V1NetworkPolicy(api_version='networking.k8s.io/v1', kind='NetworkPolicy',metadata=client.models.V1ObjectMeta(name='dns')) 221 | np.spec = client.models.V1NetworkPolicySpec(pod_selector={}) 222 | np.spec.egress = [] 223 | np.spec.egress.append(client.models.V1NetworkPolicyEgressRule(ports=[client.models.V1NetworkPolicyPort(protocol='TCP', port=53), 224 | client.models.V1NetworkPolicyPort(protocol='UDP', port=53)])) 225 | update_policy_types(np) 226 | return np 227 | 228 | 229 | def create_network_policy_peer(node: Node): 230 | if node.type is NodeType.POD: 231 | return client.models.V1NetworkPolicyPeer( 232 | pod_selector=look_up_pod_selector(node.id)) 233 | elif node.type is NodeType.IP: 234 | return client.models.V1NetworkPolicyPeer(ip_block = V1IPBlock(cidr = node.id + "//32")) 235 | else: 236 | raise Exception("Not implemented logic for NodeType:{}", node.type) 237 | 238 | 239 | def update_policy_types(np: client.models.V1NetworkPolicy): 240 | np.spec.policy_types = [] 241 | if np.spec.ingress is not None: 242 | np.spec.policy_types.append("Ingress") 243 | if np.spec.egress is not None: 244 | np.spec.policy_types.append("Egress") 245 | 246 | 247 | def create_network_policy_ports(ports: Sequence): 248 | result = [] 249 | for port in ports: 250 | result.append(client.models.V1NetworkPolicyPort(protocol='TCP', port=int(port))) 251 | return result 252 | 253 | 254 | def create_network_policy_skeleton(v: Node): 255 | assert v.type == NodeType.POD 256 | pod_data = find_pod_from_name(v.id) 257 | if pod_data is None: 258 | raise Exception("No pod data found for vertex:{}", v) 259 | # TODO fix the name of NetworkPolicy 260 | network_policy = client.models.V1NetworkPolicy(api_version='networking.k8s.io/v1', kind='NetworkPolicy', 261 | metadata=client.models.V1ObjectMeta(name=look_up_owner_name(v.id))) 262 | network_policy.spec = client.models.V1NetworkPolicySpec( 263 | pod_selector=look_up_pod_selector(pod_data['metadata']['name'])) 264 | return network_policy 265 | 266 | 267 | def look_up_pod_selector(pod_name): 268 | selector = client.models.V1LabelSelector() 269 | pod = find_pod_from_name(pod_name) 270 | if pod["metadata"]["ownerReferences"]: 271 | logger.debug('For pod, found this ownerReferences {}', pod_name, pod["metadata"]["ownerReferences"]) 272 | for owner in pod["metadata"]["ownerReferences"]: 273 | if owner["kind"] == 'ReplicaSet': 274 | rs = find_rs(owner["name"]) 275 | if rs["metadata"]["ownerReferences"]: 276 | for owner1 in rs["metadata"]["ownerReferences"]: 277 | if owner1["kind"] == 'Deployment': 278 | deployment = find_deployment(owner1["name"]) 279 | selector.match_labels = deployment["spec"]["selector"]["matchLabels"] 280 | else: 281 | raise Exception('Unknown kind:{}', owner1) 282 | else: 283 | selector.match_labels = rs["spec"]["selector"]["matchLabels"] 284 | else: 285 | raise Exception('Unknown kind:{}', owner) 286 | else: 287 | selector.match_labels = pod["metadata"]["labels"] 288 | return selector 289 | 290 | 291 | def look_up_owner_name(pod_name): 292 | pod = find_pod_from_name(pod_name) 293 | if pod["metadata"]["ownerReferences"]: 294 | for owner in pod["metadata"]["ownerReferences"]: 295 | if owner["kind"] == 'ReplicaSet': 296 | rs = find_rs(owner["name"]) 297 | if rs["metadata"]["ownerReferences"]: 298 | for owner1 in rs["metadata"]["ownerReferences"]: 299 | if owner1["kind"] == 'Deployment': 300 | deployment = find_deployment(owner1["name"]) 301 | return deployment["metadata"]["name"] 302 | else: 303 | raise Exception('Unknown kind:{}', owner1) 304 | else: 305 | return rs["metadata"]["name"] 306 | else: 307 | raise Exception('Unknown kind:{}', owner) 308 | else: 309 | return pod["metadata"]["name"] 310 | 311 | 312 | def find_pods_that_resolves_to_svc(svc): 313 | return find_pod_with_labels(svc["spec"]["selector"]) 314 | 315 | 316 | def find_pod_from_name(pod_name): 317 | return map_name_2_pod[pod_name] 318 | 319 | 320 | def find_pod_with_labels(labels: dict): 321 | for p in map_name_2_pod.values(): 322 | if labels.items() <= dict(p["metadata"]["labels"]).items(): 323 | return p["metadata"]["name"] 324 | raise Exception("Could not find pod with labels:{}".format(labels)) 325 | 326 | 327 | def find_rs(name): 328 | return map_name_2_rs[name] 329 | 330 | 331 | def find_deployment(name): 332 | return map_name_2_deployments[name] 333 | 334 | 335 | def print_graph(): 336 | print(g) 337 | 338 | 339 | def main(argv): 340 | logger.setLevel('DEBUG') 341 | inputfile = '' 342 | try: 343 | inputfile = argv[0] 344 | except: 345 | print('3-analyse.py ') 346 | sys.exit(2) 347 | process_file(inputfile) 348 | 349 | 350 | if __name__ == '__main__': 351 | main(sys.argv[1:]) 352 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/4-delete-sidecar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source env.sh 4 | DEPLOYMENTS=$(kubectl get deployment -n $TARGET_NS -o name) 5 | 6 | for d in $DEPLOYMENTS 7 | do 8 | INDEX=$(kubectl get $d -n $TARGET_NS -o json | jq '.spec.template.spec.containers | map(.name == "tcpdump") | index(true)') 9 | kubectl -n $TARGET_NS patch $d --type=json -p="[{'op': 'remove', 'path': '/spec/template/spec/containers/$INDEX'}]" 10 | done 11 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Generating Kubernetes Network Policies By Sniffing Network Traffic 3 | tags: ['kubernetes', 'NetworkPolicy','automation','network'] 4 | status: draft 5 | --- 6 | 7 | # Generating Kubernetes Network Policies By Sniffing Network Traffic 8 | 9 | This blog post is about an experiment to automate creation of [Kubernetes Network Policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/) based on actual network traffic captured from applications running on a Kubernetes cluster. 10 | 11 | **All the code referred in this blog post can be found [here](https://github.com/mcelep/blog/tree/master/automated-networkpolicy-generation).** 12 | 13 | *We worked on the blog post idea with a VMware colleague [Assaf Sauer](https://ch.linkedin.com/in/assaf-sauer-b6261b23).* 14 | 15 | ## But why? 16 | 17 | [Network Policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/) are used for allowing/blocking network traffic of applications running on Kubernetes clusters. Enterprises which process critical customer data such as financial institutions(Banks, Insurances,etc. ) have quite strict security requirements and those that run applications on Kubernetes clusters are very likely to use *Network Policies*(or depending on the CNI plugin they use, something similar to it such as [Antrea's ClusterNetworkPolicy](https://github.com/vmware-tanzu/antrea/blob/a9adf5a58acee9bfb06fbe75b0f2d097ec362c99/docs/antrea-network-policy.md) or [CiliumNetworkPolicy](https://docs.cilium.io/en/v1.8/concepts/kubernetes/policy/#ciliumnetworkpolicy)) to control which ingress/egress network traffic is allowed for applications. 18 | 19 | For simple applications such as a classical 3-tier architecture app that consists of a frontend, a backend that connects to a database, creating *Network Policies* is straight-forward. However, more complex applications that consists of many components(think about micro-service architecture based applications with many components) coming up with the right network policies that are precise (what we mean by precise here is, communication between two components is allowed only if there's an explicit need and it's allowed only on known ports) can be quite time consuming. And this is why we came up with a couple of scripts to make it easier to create standard(non CNI specific) Kubernetes Network Policies. 20 | 21 | ## How? 22 | 23 | The idea behind the automation of network policy generation is simple. The diagram below summarizes it: 24 | [![Big picture](https://github.com/mcelep/blog/blob/master/automated-networkpolicy-generation/system-overall.png?raw=true)](https://github.com/mcelep/blog/blob/master/automated-networkpolicy-generation/system-overall.png?raw=true) 25 | 26 | It consists of the following steps described below: 27 | 28 | ### 1) Capture network traffic 29 | 30 | First of all, we need a way to capture network traffic of each pod running. We use the good old [tcpdump](https://en.wikipedia.org/wiki/Tcpdump) for capturing the traffic and the [kubernetes sidecar pattern](https://www.google.com/search?q=kubernetes+sidecar+pattern). For this to work, we need the right privileges for the pod/container that will run the [tcpdump image](https://hub.docker.com/r/dockersec/tcpdump) i.e. tcpdump container needs to run with root user. We are aware that this might be a no-go in highly secure enterprise kubernetes installations. That said, the envisioned usage for the scripts in this repo is to run the apps in a cluster that: 31 | 32 | - Does not enforce Network Policies at all(or all ingress/egress traffic is allowed by using a Network Policy such as [this](https://kubernetes.io/docs/concepts/services-networking/network-policies/#default-allow-all-ingress-traffic) and [this](https://kubernetes.io/docs/concepts/services-networking/network-policies/#default-allow-all-egress-traffic)) 33 | - Allows a pod to run with user root 34 | 35 | In the case where policy generation happens on one cluster and application runs in a different cluster, you will need to make sure that the IPs,FQDNs specified for Network Policies are adjusted accordingly for the target environment. For example if there is an egress Network Policy that is used for connecting to a Oracle database running on IP 10.2.3.4 and if this IP is different on the actual target environment where the application would be deployed, you will need to adjust that IP. 36 | 37 | ### 2) Generate network traffic 38 | 39 | In order to create the right Network Policies, we need to capture network traffic that would represent all the use cases, thus all potential network communication related to the application. It's up to you to make sure that during the time the network traffic is captured, you generate meaningful load on the application. For example, if your application is a REST based web application, you would make sure that you hit all the relevant REST endpoints. If the application does some processing based on messages received from a MessageQueue, it's up to you send those messages to the queue so that the applications is performs what it would normally perform in production. 40 | 41 | ### 3) Collect capture files and Kubernetes data 42 | 43 | Tcpdump sidecar container runs tcpdump with the following command: ```tcpdump -w /tmp/tcpdump.pcap```. So before we can analyse the traffic, we need to collect all the pcap files from pods. Moreover, we also need some metadata from Kubernetes. We will capture information about Kubernetes resources such as Services, Deployments, ReplicaSets so that later we can use that to analyse which IP is owned by which application. 44 | 45 | ### 4) Analyse data & Generate Network Policies 46 | 47 | After collecting all the packet capture and kubernetes data about IPs, Ports, Labels now we can build a graph. In this graph, nodes/vertices would represent pods and edges would be the network communication between these pods. In the python script that generates the network policies, first a graph is build and in the second part edges are traversed and for each edge a Network Policy is generated. 48 | 49 | ## Let's see it in action! 50 | 51 | Let's go through the steps below and see if we can generate network policies for a test application. The test application is a micro-service demo application from Google that can be found [here](https://github.com/GoogleCloudPlatform/microservices-demo). 52 | 53 | 54 | ### Environment preparation 55 | You need access to a Kubernetes cluster and you will need the following tools installed: 56 | - Kubernetes CLI 57 | - Python (v3) 58 | - Pip(to manage python dependencies) (v3) 59 | - Tshark which is a Terminal-based [Wireshark](https://www.wireshark.org/) (see [here](https://www.wireshark.org/docs/man-pages/tshark.html)) 60 | - [Cut](https://man7.org/linux/man-pages/man1/cut.1.html) 61 | - [Jq](https://stedolan.github.io/jq/) 62 | 63 | To install python dependencies, run the following command from top project folder: ```pip install -r requirements.txt``` 64 | 65 | We've tested the scripts in this repo on Mac OS and Linux. 66 | 67 | ### Deploy a test application 68 | 69 | ```env.sh``` file on the top-level project folder contains some variables that are used in multiple scripts. Please override those variables with your preferred values, e.g. **TARGET_NS** is the Kubernetes namespace which will be used for all scripts. By default *TARGET_NS* is set to *netpol-demo*, you can edit it if you like. 70 | 71 | The excerpt below is used to create a Kubernetes namespace and then deploy a test application. 72 | The test application in this case is a microservice demo application that can be found in this [repo](https://github.com/GoogleCloudPlatform/microservices-demo). You can however use any application that you fancy. 73 | 74 | ```bash 75 | source env.sh 76 | # create a namespace if it does not already exist 77 | kubectl create ns $TARGET_NS --dry-run -o yaml | kubectl apply -f - 78 | # deploy application 79 | kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/cbd3c9643400de7471807821e212a6f3db38ed14/release/kubernetes-manifests.yaml -n ${TARGET_NS} 80 | ``` 81 | 82 | ### Patch deployments in $TARGET_NS 83 | Running the following command: ```./1-inject-sidecar.sh ``` patches all the deployments and injects a sidecar container that runs tcpdump with the following command: ```tcpdump -w /tmp/tcpdump.pcap```. In our test application, we only have pods that are controlled by deployments. If you are to use a different application that the demo google app used here, you will need to adopt the sidecar injection mechanism. 84 | 85 | ### Prepare to copy capture and metadata 86 | 87 | Notice that the script below(*2-copy-capture-and-metadata.sh*) will capture pod names and then it will print out that it will wait for *TEST_DURATION_IN_SECONDS*. During this duration, you should 'load' the application so that network traffic that covers all the potential communication patterns in a real-world use case is generated. 88 | 89 | Run the command below: 90 | 91 | ```./2-copy-capture-and-metadata.sh``` 92 | 93 | 94 | If you run into a issue while running this script, you can re-run it. Bear in mind, though that you will need to make sure that app traffic is generated also during the re-run. 95 | 96 | ### Generate traffic 97 | In the case of our demo application, there is pod called *load-generator* that will be deployed with the rest of the other app components. This pod keeps calling the ui to generate application flows. You can see the logs from *load-generator* pod by running: ```kubectl -n $TARGET_NS logs $(kubectl get pods -l app=loadgenerator -o jsonpath='{.items[0].metadata.name}' -n $TARGET_NS) -c main```. 98 | 99 | You don't need to do anything extra if you use the same Google microservice demo application to generate network traffic. However if yo use another application, you will need to run your own load generation tool while the script *./2-copy-capture-and-metadata.sh* keeps printing out that message: 'Going to wait for X seconds so that application traffic can be generated...' 100 | 101 | ### Analyse data & build NetworkPolicies 102 | Run the command below to analyse the data and generate NetworkPolicies. The input to the script is a capture json file that is prefixed with *capture-*. You can find candidate capture file(s) by running ```ls .tmp/capture-*.json```. 103 | 104 | ```bash 105 | ./3-analyse.py .tmp/capture-XXX.json 106 | ``` 107 | 108 | Generated NetworkPolicies should be in *.tmp/network-policies* folder. For our demo application, we generated one NetworkPolicy file for each application including both ingress and egress rules. Moreover, a DNS policy that allows egress communication on port 53 is created by default. 109 | 110 | There's also a graph(in [DOT](https://graphviz.org/doc/info/lang.html) format) generated as a result of running the *3-analyse.py* script. You can use an online tool such as [this](https://dreampuf.github.io/GraphvizOnline) or [this](https://edotor.net/) to create a visual representation of the graph. Here are some example images created via dot representation. 111 | 112 | [![Micro-service demo app graph](https://github.com/mcelep/blog/blob/master/automated-networkpolicy-generation/graph_ms_demo.png?raw=true)](https://github.com/mcelep/blog/blob/master/automated-networkpolicy-generation/graph_ms_demo.png?raw=true) 113 | 114 | [![Simple demo app graph](https://github.com/mcelep/blog/blob/master/automated-networkpolicy-generation/graph_simple.png?raw=true)](https://github.com/mcelep/blog/blob/master/automated-networkpolicy-generation/graph_simple.png?raw=true) 115 | 116 | ### Clean-up 117 | 118 | The tcpdump sidecar keeps capturing network traffic as long as it's running. So don't forget to remove those sidecar containers after you're done. Run ```./4-delete-sidecar.sh``` to get rid of the tcpdump sidecar containers. 119 | 120 | ### Gotchas 121 | 122 | Scripts in this repo were created in a proof of concept setting, code quality has room for improvement. There are some limitations: 123 | 124 | - Scripts were developed to generate NetworkPolicies for applications that runs pods running in a single namespace. So if there are calls happening to other namespaces than where the application is running, network policies will not be correctly handled. 125 | 126 | - Network policy rules for traffic to/from outside of cluster is IP based. If the applications communicates with a domain that is backed my multiples IPs, there will be NetworkPolicy rules only for the IPs that were resolved during the time the scripts ran. So, you might end up missing some rules. 127 | 128 | - Only TCP traffic is analysed. There's a NetworkPolicy for DNS generated but there will be no other UDP based NetworkPolicies created. 129 | 130 | - Running tcpdump requires some elevated privileges. In an environment/cluster where NetworkPolicies would be enforced, the chances are that you can't get those elevated privileges. We think, however, that the concept/scripts provided in this blog post could be beneficial in some use cases such as COTS(commercial off-the-shelf) software for Kubernetes. In a COTS software on Kubernetes scenario, a Vendor could benefit from generating NetworkPolicies in a cluster that is not very restrictive from a security perspective. Generated NetworkPolicies could then be provided as a template to the software vendor's customers. 131 | 132 | Feel free to extend these scripts to cover more use cases :) 133 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/create-capture-metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | from datetime import datetime 5 | import json 6 | import subprocess 7 | 8 | 9 | def process_pods(ns, pod_names): 10 | capture_metadata = {"pod_metadata": []} 11 | for pod in pod_names: 12 | capture_metadata["pod_metadata"].append({"pod": pod, "file": '{}.pcap'.format(pod), "IP": find_pod_ip(ns, pod)}) 13 | 14 | capture_metadata["services"] = json.loads(lookup_service_data(ns)) 15 | capture_metadata["pods"] = json.loads(lookup_pod_data(ns)) 16 | capture_metadata["rs"] = json.loads(lookup_rs(ns)) 17 | capture_metadata["deployments"] = json.loads(lookup_deployments(ns)) 18 | 19 | capture_metadata_filepath = ".tmp/capture-{}.json".format(datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) 20 | with open(capture_metadata_filepath, "w") as a_file: 21 | a_file.write(json.JSONEncoder().encode(capture_metadata) + '\n') 22 | 23 | 24 | def find_pod_ip(ns, pod): 25 | return subprocess.check_output("""kubectl -n {} get pods {} -o jsonpath='{{.status.podIP}}' """.format(ns, pod), 26 | shell=True, timeout=60, stderr=None).decode("utf-8") 27 | 28 | 29 | def lookup_service_data(ns): 30 | return subprocess.check_output("""kubectl -n {} get svc -o json""".format(ns), 31 | shell=True, timeout=60, stderr=None).decode("utf-8") 32 | 33 | 34 | def lookup_pod_data(ns): 35 | return subprocess.check_output("""kubectl -n {} get pod -o json""".format(ns), 36 | shell=True, timeout=60, stderr=None).decode("utf-8") 37 | 38 | 39 | def lookup_rs(ns): 40 | return subprocess.check_output("""kubectl -n {} get rs -o json""".format(ns), 41 | shell=True, timeout=60, stderr=None).decode("utf-8") 42 | 43 | 44 | def lookup_deployments(ns): 45 | return subprocess.check_output("""kubectl -n {} get deployments -o json""".format(ns), 46 | shell=True, timeout=60, stderr=None).decode("utf-8") 47 | 48 | 49 | def main(argv): 50 | if len(argv) == 0: 51 | print('Provide at least one argument / pod name!') 52 | sys.exit(2) 53 | process_pods(argv[0], argv[1:]) 54 | 55 | 56 | if __name__ == '__main__': 57 | main(sys.argv[1:]) 58 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/env.sh: -------------------------------------------------------------------------------- 1 | TARGET_NS=netpol-demo 2 | TEST_DURATION_IN_SECONDS=30 3 | -------------------------------------------------------------------------------- /automated-networkpolicy-generation/graph_ms_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/automated-networkpolicy-generation/graph_ms_demo.png -------------------------------------------------------------------------------- /automated-networkpolicy-generation/graph_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/automated-networkpolicy-generation/graph_simple.png -------------------------------------------------------------------------------- /automated-networkpolicy-generation/requirements.txt: -------------------------------------------------------------------------------- 1 | graph-tools 2 | kubernetes -------------------------------------------------------------------------------- /automated-networkpolicy-generation/system-overall.drawio: -------------------------------------------------------------------------------- 1 | 7Vxtc5s4EP41/hgGSUiIj7Ed997ay0xu5tr7coONYnPBhgOcOP31J4GwLSFj4oKd5OJ2WrQIAbuPVs9KKwZotNx8Sv1k8TkOWDSAdrAZoPEAQoQp4P8JyXMpARDapWSehoGU7QR34XcmhVW1dRiwTKmYx3GUh4kqnMWrFZvlisxP0/hJrXYfR+pdE3/OaoK7mR/VpX+GQb6QUoKd3YmfWDhfVLcGxCvPLP2qtnyVbOEH8dOeCN0M0CiN47w8Wm5GLBLqqxRTXjc5cHb7ZClb5W0umH+///L38p+fJv9+voH4r9sg3DxcYVo28+hHa/nK8mnz50oHabxeBUy0Yg/Q8GkR5uwu8Wfi7BM3O5ct8mXES4AfRv6URUN/9jAvLhvFUZzyU6t4xesP78MoqkQDiDz+m0yEPF7l0vjAleW9enbx4/KY3zfMBZiIKNZVILXyyNKcbfZEUiWfWLxkefrMq8izVwDY0kASo6QqP+0Z3KZSuNgzNnGk0Jcom2+b39mBH0hTvMAsBBjMQiJ+32EQPvLDeV68finKEn+lmIz8uxaoKtR4lRV6veYVAE02u5NVK7+upyxdsZx3M2iPonWWs7RqmT982bh6Qy5WHkMDDFd9rqIiy9P4gTWAQYr8KJyveHHGDcqfAg2FIUPeHa/liWUYBOI2RhiqQD2CxMshjkAVcJg4NcA5BrjBDtD21+8PV5n3y9d1fv3NAz8/TNHn71fOQbAJrZyOrOskibjx8jAW8PziL1lWGGyHorL9/yuMapgxIKsBRtSzPA97NiDIQ47jKKACgAKL2oS6uAYuRC1CKYaOvNgxYA1ZmFLX9ojrOsBDmPSEPfSS8QccH3+2o6xdH27G+IaOnY7UD2yijRuwPm642NCPcRfDhlGZAB7XpoBxUtfNEBJSDMVbBYImRUm+5U+rZu0TFAhty7aB43KFeBRVY16lTizOCg5ZwpDWlAsdgXAEXSrRTAw4Bq5FbEydCsdOT6rHZ8TxZHJDRqO2OG5ASt04Ter1LqjeOk0dcU/rh6sdV5mmOk0JdcF0T+AvhcYL8R2PMWZ+Wg5lf8ySYM3PVTV04jM1kKHw4BjWmc2HxZ9ebF4142DLpoB3NQgQEN1SG1As13O4hSkqOySpdUjqWrbjgR1o6oDhXtMCvI4nGwI94QUcJtCtOY1j4jS3cfDjgBvFy4Tzh+I5xK1OBdsHj+qrMxBoUYUjqX3BJZYHMOJjOadYDgZ1jgUAsWyCsYtct+gUF+RYpE/nqWH5dXnN0egGF6ymN6B41s5fChpDX+o0AcTWFiSF5zR4TYwsxAkagJ5dgK4vr9mCjVf88Ww8kdB3RBOBKdZ+zTwRvS2eCOo8/IModmX0d0kU64PjKyWKw1OR9sES++oJ74olco2eiSaejOTXTBObkfK+aKLXlib2SAs9u4kWeuhN0cLq/d8MLfTeFi2E9emgD1rYldHfIy2s+t/rp4WjD1p4flrY3BPeFS2E9TmhnmjhyUh+zbSwGSnvihbCDkJpYx5JDUNjP/d57bs8TtkRmDQ6N278RBwGvL2saOyMeDm6PO6q+QUAmRLTgCHBAPSWKQTrMWJNqWwVXIvMS+H6Iz/LwpmqQUMOoOdxOduE+VehWAtiWfwmLxHH441UelF4loXKgvcR28i7DrmG0+eyJVwVv1UXi8KuqaJUtVW+CQtqGaG1ZK8sXqcz1sJp5n46Z03+oewzjUTamEIiZSmL/Dx8VB/YZHV5h9s4LDpiBTKk5T4C29agU76qvG6HnnpTHtX4Hh941aZKZdSaKoC4ffUfwGaLqPRkbG4hpQBKIrUBU52BukNwkrbYPD733yc2qeb/sI6n1tCkWsKlQ9shk+vef96rlogKWcMje65K9TyspGTzg7LJTnGPWswd7EE88LOFmsydqYOd1gd4VIZvYI338zP3xY+fkXrhQjzkf8XMC/cFI9siSPw3wLz2qDgBCjHUpJ5RWjSh1/QONOwWV/PzhkaAJoNUq8v/7jrdcjMXexmsh21atBXO4pV4zfTn4mDsJ6HOBLoY8Tk9U9EDiYVAbcg35wZD3NOQj0yRsAYvP0vKzRf34UZga9+lquAYmwLDcFnswlDiOLiTj8PlnD96FE7FC2Qzn3O+yS5r3coe5+clZpqdeLRmyPw07RdwLdiXlVrkH344gS6dQPY468MJeFRzAgbSbxp1nb6iOmSK6rqmVUClVW5rWrXHqarLeqFVF2NBxLV58C7y+yERhAIp6HCpa7keIC4qZwg0ELRnSNSxsCc3EvAbgZfcpWdej1rEnB+u7Q24NtfxLKrh17mwd3NasJvelzIJaFzKhJ0tZfa1WoJbsI8f12KDAQ/rFukTHHXtQROf7lNdzjk3PU0mcrBt1YUbzNu82nt+JbYYFnrC3CF9bKruaFF9M55VzQzsz92KoV2uThU/wz48QGnh+/tRYYvN3311W/copEr1nFslJk/2atZ9f4t93or9ia1Y6udx+rH0e/al37LPHO77yALe/o+ojoBgq0qy34e6427F+iKv1xfSe90V0hVS+xoVO1jaPYSENi5MrMiivgzbYh3kI156A/GSw4NyNRSgLjXNCJ81YsKmnZWXng8yLwMH1Sd75DJd9X0hzkodxUlX80ZfB8o00nbR+OCU0qDl9FGLVTmn7bIcOE6del2W05aM3SqkevGk0/bjCNVWLqK11NWynKOtJFJ0hmU53CI+qWjRb4Lj3MZZWHwqBY2ncZ7HS7XP6BQqjxMT0zpK0fZzQXUf539fp8zKZmmY5H/zi1jlzFl688hKn14bWbtwddjRXR1Ahs9pmLNddAB2t2+2RXy05+qksrcDKlANKOS3fs51uSok0EZbe20dVZNz3Dqo9pkEe6kwPLpUfOqufGiOXZRuWRpyZQpode3waEt/h1+Zv/NO9XcYaHNRSE+26XkmnbRghhfAs7KG07yEIwo6JPcwDpV8L3AE4B1iuXL2x3NqLruaRPUh9/ScGq1b4JY5NV1huRrSXwmWD6QcHoOg2ac3c05TH7gElMHxWeE+oex4GmGAJztmV/+SV19EFDnaCFClW/bKRN2XRWxncvugO7cPKFK5DUVHO17P5KZ1L0L24JK9CEGdk+jYb92LYC1fXSfmfQ8Jh79juQCGaca6UEgME/xiIvxKRlpihl8GWwf3KTR/j/VJmlG0tIrTpR/VW7pe8/iv+nBmGY+Jr7Pepzwq5KBahKI0jeJ5UfKFGZI0nrEsK77imsz8RFTnEZwo+quA/6t855Xr2g/KXRR5LF4pZX7O+MEXlj/F6cNtHIWz510bjZ+DrU8Ta9rVnM3pCwtiajJczbkA70p/iEh4LPbQmqeEYx6b3keFb1uEQcDE3J26xtB5PEtQLZ51UD2e3Q5Z6uTwy+NZXtx9SLrsT7sPcqOb/wA= -------------------------------------------------------------------------------- /automated-networkpolicy-generation/system-overall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/automated-networkpolicy-generation/system-overall.png -------------------------------------------------------------------------------- /automated-networkpolicy-generation/tcpdump-sidecar-patch.yaml: -------------------------------------------------------------------------------- 1 | spec: 2 | template: 3 | spec: 4 | containers: 5 | - name: tcpdump 6 | image: docker.io/dockersec/tcpdump 7 | command: 8 | - "/bin/bash" 9 | - "-c" 10 | - "tcpdump -w /tmp/tcpdump.pcap" 11 | -------------------------------------------------------------------------------- /debug-k8s-workload-1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginxinc/nginx-unprivileged 2 | 3 | COPY /viewer/ /usr/share/nginx/html 4 | -------------------------------------------------------------------------------- /debug-k8s-workload-1/skaffold.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: skaffold/v2beta23 2 | kind: Config 3 | build: 4 | local: 5 | push: false 6 | artifacts: 7 | - image: k8s-debug-flow-viewer 8 | context: . 9 | hooks: 10 | after: 11 | - command: ["sh","-c","open http://localhost:8080"] 12 | os: [darwin] 13 | deploy: 14 | docker: 15 | images: 16 | - k8s-debug-flow-viewer 17 | portForward: 18 | - resourceType: Container 19 | resourceName: k8s-debug-flow-viewer 20 | port: 8080 21 | localPort: 8080 -------------------------------------------------------------------------------- /debug-k8s-workload-1/viewer/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Kubernetes application debugging flow 4 | 5 | 26 | 27 | 28 | 29 |
30 |

Kubernetes application debugging flow

31 |
32 | 33 |
34 |
35 |
36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 82 | 83 | -------------------------------------------------------------------------------- /debug-k8s-workload-1/viewer/resources/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/debug-k8s-workload-1/viewer/resources/screenshot.png -------------------------------------------------------------------------------- /network-policies/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Lifecycle of Kubernetes Network Policies 3 | tags: ['kubernetes', 'NetworkPolicy','automation','network'] 4 | status: draft 5 | --- 6 | # Lifecycle of Kubernetes Network Policies 7 | 8 | In this blog post, we will talk about the whole lifecycle of Kubernetes Network Policies covering topics such as creation, editing, governance, debugging and we will also share insights which can create better user experiences when dealing with Network Policies. 9 | 10 | *[Andy Knapp](https://www.linkedin.com/in/andy-knapp-6a72b6108/) has reviewed this article and suggested some changes. Thanks a lot Andy!* 11 | 12 | ## Enter Network Policies 13 | 14 | As Kubernetes adaption continues to grow in large enterprises, security relevant aspects of Kubernetes such as Network Policies, which lets you control what network resources are allowed to be accessed from/to Pods, become more important. 15 | 16 | Kubernetes is a very powerful platform and with all that power some complexity is also introduced. Especially folks who've just started to learn about Kubernetes, can get easily overwhelmed due to all the new things they will need to interact with and master. Kubernetes after all is supposed to help companies become more nimble but a platform alone can't fix most typical problems of enterprises such as culture, processes and silos. Even if your company has the best Kubernetes platform, if you don't integrate it well to the rest of the IT ecosystem of an Enterprise, you will never be able to get the full benefits of a Kubernetes platform. 17 | 18 | Unfortunately, many large enterprises that adopt Kubernetes, suffer from the fact that the organization as a whole is not very agile. In the average large enterprise today, unfortunately there is still a lot of manual work that needs to happen for any IT related order/change to complete. Often Network/Security teams are isolated from the Platform & Application teams from a organizational perspective and the 'silo' mentality together with each organizational unit having its own targets & incentives, security related topics often become big sources of problems and slowness. 19 | 20 | Network Policies, which play a very critical component for network security, are only as good as the user experience around them such as how easy it is to create them? how easy it is to have them approved & applied? In the rest of this blog post, we will talk about some ideas that target different stages of Network Policies that should help you to create an optimal user experience around using Network Policies. 21 | 22 | ## A Pattern: Network Perimeter Security Delegation to Kubernetes 23 | 24 | One pattern that helps a lot in many Enterprises, is opening communication (at least for some common services such as a logging service, monitoring service) all the way to perimeters of the Kubernetes Clusters and then let Network Policies control the network traffic. When a team needs to manage Network Policies and at the same request new Firewall rules via different mechanisms such as ServiceNow or some other workflow/ticketing system, the total turnaround time for getting the Network access just takes too much time. The main motivation behind this pattern is doing the lengthy Firewall changes just once and on a Kubernetes platform level and rely on Network Policies for the fine-grained control. 25 | 26 | ![A Pattern: Network Perimeter Security Delegation to Kubernetes](https://github.com/mcelep/blog/blob/397bbb672a302f1a0b4b9dcf9883912d258ceebc/network-policies/network_perimeter_kubernetes.png?raw=true) 27 | 28 | When the Network Perimeter Security is controlled on the Kubernetes cluster level, the default allowed traffic will need to be controlled carefully i.e. you will probably want to limit what ingress / egress network traffic is allowed by default. One should bear in mind that, when no Network Policy is applied on a namespace, Kubernetes exercises no control for the network traffic. This can be done in different ways depending on the [CNI(Container Network Interface)](https://github.com/containernetworking/cni) plugin used in a Kubernetes cluster. One way of doing it would be to apply a Network Policy that blocks most of the egress/ingress traffic for all Namespaces and control RBAC(role based access control) in such a way that non-admin users of Kubernetes clusters can't create/edit/delete Network Policy objects themselves without some governance. Based on the software that implements CNI plugin that your clusters use, you might also need to limit what network access each namespace gets by default. Using [NSX](https://www.vmware.com/products/nsx.html) firewall rules to implement this idea is something we know from the field at VMware Tanzu Labs. 29 | 30 | At the same, for such a pattern to work successfully, the turnaround time for applying Network Policies should be as short as possible. In the Governance section of the post, a couple of ideas about how governance can be shaped so that Network Policies can be accepted or rejected as quickly as possible. 31 | 32 | ## Creating Network Policies 33 | 34 | ### Basics 35 | 36 | It's key to have a basic understanding about Network policies, and the two documents below are very good starting points: 37 | 38 | - [Kubernetes official documentation about Network Policies]( https://kubernetes.io/docs/concepts/services-networking/network-policies/) 39 | - [A tutorial from Kubernetes Network Policy Community](https://github.com/networkpolicy/tutorial) 40 | 41 | Once you've understood the basics, it's probably time to get some hands-on experience. See if you can install your favorite CNI plugin that supports Network Policies on [minikube](https://minikube.sigs.k8s.io/docs/) or another Kubernetes cluster you can access. If you have to install a CNI plugin yourself, you would require admin rights. It's important here to note that although Network Policy is an official part of Kubernetes and a standard Kubernetes resource, it's the CNI plugin that actually 'implements' the Network Policies, so as long as a CNI tool supports Network Policies, you should be able to test the basic functionality of Network Policies. That said, vanilla Kubernetes Network Policies have certain limitations as explained [here](https://kubernetes.io/docs/concepts/services-networking/network-policies/#what-you-can-t-do-with-network-policies-at-least-not-yet) so it makes a lot of sense, if possible, to test out Network Policies using the actual CNI plugin you would use in production. 42 | 43 | If you desire to see a good catalogue of example Network Policies that cover most of the use cases, check out [this github repo](https://github.com/ahmetb/kubernetes-network-policy-recipes). 44 | 45 | Another important point to consider is how you deal with pod selectors for Network Policies. A key called ```spec.podSelector``` controls to which policies the Network Policy will be applied on. When the key is empty, i.e. ``` podSelector: {}``` , it applies to all pods in the namespace where the Network Policy is created. If you want to target specific pods, you can do it by using a pod selector such as: 46 | ``` 47 | podSelector: 48 | matchLabels: 49 | app: bookstore 50 | ``` 51 | If you have lots of NetworkPolicies to write because your app has many components and if you want to write a policy that controls traffic as precisely as possible meaning writing pod selectors that only targets the pods where a specific egress/ingress rule is required, you can use a key called **matchExpressions**. See the example below which helps you to target two different kinds of pods via labels *app=bookstore* or *app=database*: 52 | ``` 53 | podSelector: 54 | matchExpressions: 55 | - {key: app, operator: In, values: [bookstore, database]} 56 | ``` 57 | 58 | #### DNS 59 | 60 | Forgetting adding a **Network Policy for DNS** calls is another common mistake. Depending on how your pod is configured as explained [here](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy), application pods might need to communicate to the DNS server running on your cluster or a DNS server running outside. Make sure to add a Network Policy to accommodate DNS communication needs to your namespace. If your CNI plugin allows it you can do also apply a DNS policy on the cluster level, e.g. with [Antrea's ClusterNetworkPolicy](https://github.com/vmware-tanzu/antrea/blob/28ef522ced7045f567c22b916cb29d9272f9c92b/docs/antrea-network-policy.md). 61 | 62 | #### Pod A's Egress is Pod B's Ingress 63 | 64 | If **Pod A** needs to call **Pod B**, you will need to create a Network Policy that has **egress** rules for **Pod A** and another Network Policy that has **ingress** rules for **Pod B**. It's quite common to forget this symmetry requirement between egress & ingress rules for communication between Pods. 65 | 66 | 67 | ### Editing Network Policies 68 | 69 | "Kubernetes and YAML in everybody's mind really go together" says Joe Beda, one of the co-creators of Kubernetes, in [this talk](https://www.youtube.com/watch?v=8PpgqEqkQWA). And he's right, it's the recommended way of writing configuration files as you can see [here](https://kubernetes.io/docs/concepts/configuration/overview/) and most of the folks that I've worked with use YAML (instead of JSON). If I got a penny for every time I saw someone break indentation in a YAML file, I would be rich by now :). So do yourself a favor and use an editor with a plugin that helps you notice/fix these issues easily. Basic YAML syntax support will definitely help a lot, if you can get a plugin that also understand Kubernetes YAML syntax even better! Improved Kubernetes YAML editing experience will not only help for Network Policies obviously, you can improve the whole Kubernetes experience by a more pleasant resource editing experience. Here are some tips: 70 | 71 | - [Here](https://www.youtube.com/watch?v=eSAzGx34gUE) is a video and [here](https://octetz.com/docs/2020/2020-01-06-vim-k8s-yaml-support/) is a blog post that shows how you can do it with Vim. 72 | 73 | - If Visual Studio Code is your editor of choice, you might want to look into [this plugin](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) 74 | 75 | - For most of the editors you should be able to get at least YAML syntax support so go ahead install a plugin or update your editor config so that YAML support is activated. 76 | 77 | [Cilium editor](https://editor.cilium.io/) is a tool to create the Network Policies with a graphical editor. Especially when beginning writing Network Policies, the visual interaction might be very helpful. Below is a snapshot from the cilium editor: 78 | ![Cilium Editor](https://github.com/mcelep/blog/blob/397bbb672a302f1a0b4b9dcf9883912d258ceebc/network-policies/cilium-editor.png?raw=true) 79 | 80 | For an application that includes many different kinds of components e.g. a micro-service application, writing a lot of Network Policies manually can be quite cumbersome. If your application is COTS(Commercially Available Off the Shelf) software, perhaps the software should provide all the Network Policies and keep them up to date as the Software and its components evolve. 81 | 82 | [This blog](https://itnext.io/generating-kubernetes-network-policies-by-sniffing-network-traffic-6d5135fe77db) talks about an idea about how you can automatically generate Network Policies based on actual application network traffic. 83 | 84 | ## Governance 85 | 86 | Although Network Policies are well defined Kubernetes resources, there might be certain things that your company wants to enforce that is specific to them due to regulatory/auditing purposes. For example, certain annotations that might provide more information about the context of a Network Policy might be a thing security department wants to enforce, e.g an annotation such as *k8s.example.com/projectId* in metadata section of the Network Policy. In such a case, it becomes critical to document the expectation, provide examples of good Network Policies and let platform users know about the documentation. 87 | 88 | 89 | ### Git Pull Requests To The Rescue 90 | 91 | 92 | In most of the environments where you would require Network Policies, there will typically be a process around getting the Network Policies applied on a kubernetes cluster. Such process could be some sort of an approval/auditing process and users will not be authorized to edit Network Policies themselves directly. One very common way of doing this is implementing an audit process based on Git repositories. The typical flow in such a setup looks like this: 93 | 94 | - A developer checks out the Git repo where the Network Policies reside 95 | - Developer creates a new branch and works on editing/creating Network Policies 96 | - Developer creates a pull request based on his branch 97 | - Security experts need to approve or reject the Pull request and provide enough information about why if they reject a pull request 98 | - If a pull request gets approved, a pipeline picks up the change and applies it on the cluster, developer gets notified. 99 | 100 | The steps above capture the gist of a Git pull request process. Compared to a traditional model in which there would be a ticket opened in a system such as ServiceNow, there might be some improvements. For example, instead of using some free text or some other format to capture data, users end up creating the Network Policy in a way that can be directly used on Kubernetes clusters. Ideally, automation should be used as much as possible to make sure Network Policy requests are processed as correctly and as quickly as possible. 101 | 102 | In the next section, we will go into details of how Network Policies creation can be fully automated without any human interaction and GIT. Using something like [Gatekeeper](https://github.com/open-policy-agent/gatekeeper) and security folks letting the whole process be fully automated, is sometimes too big of a conceptual change i.e. folks who are responsible for security must understand and potentially collaborate in the implementation of this automation. Using a Git based approach with at least some automation, might already give you loads of benefits and improve the time to production significantly. Ideas below could be embedded into your Git Pull Requests based process: 103 | 104 | - It's very easy to make mistakes when writing yaml files, so the first thing to do is to make sure that incoming Pull requests contains valid Network Policies. Use ```--dry-run``` parameter with ```kubectl apply``` or ```kubectl create```, to see if a Network Policy resource file is valid. The earlier you catch a validation error in the process, the cheaper it is to fix. 105 | 106 | - Maybe you can't automatically approve all the incoming Network Policies via automation but you can at least use some useful insights about the incoming network profile by checking if the IP ranges are allowed, if there are any cross namespace requests, etc. So it might need to be a human that needs to do final approval of a Network Policy, but his work can be simplified by relatively cheap automation. [OPA(Open Policy Agent)](https://www.openpolicyagent.org/docs/latest/) with its [Rego](https://www.openpolicyagent.org/docs/latest/#rego) DSL can help here to write some automation or simply use your favorite (scripting) language to parse Network Policy files and evaluate them towards a bunch of rules that you create based on your organization's needs. 107 | 108 | ### OPA/Gatekeeper based fully automated control of Network Policy creation 109 | 110 | [Open Policy Agent(OPA)](https://www.openpolicyagent.org/) is a Cloud Native Computing Foundation project and it aims at solving 'policy enforcement' problem across the Cloud-Native stack(Kubernetes, Docker, Envoy, Terraform, etc.). OPA comes with a language called [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) to author policies efficiently and other than Rego, there are a number of tools & components in OPA to make use of policies easier & efficient. 111 | 112 | The quickest way auditing Network Policies automatically can be implemented directly on a K8S cluster. OPA comes with a component called [Gatekeeper](https://www.openpolicyagent.org/docs/latest/kubernetes-introduction/). Gatekeeper is an Admission Controller, and it aims at managing and enforcing policies easily on K8S clusters. That is, one can hook into [Admission Controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) mechanism of Kubernetes to reject K8S Resource modifications that are not allowed by policies. By writing *Rego* policies that control what Network Policies are allowed and rejected, a developer will have very quick feedback regarding which network traffic is allowed. OPA and Gatekeeper are very interesting technologies that are getting popular in the Kubernetes ecosystem and it's definitely worth to invest a bit of time to understand it better. We are planning to provide a more practical example of how you can use Rego to control Network Policies in another article. 113 | 114 | ## Debugging 115 | 116 | Another important stage while using Network Policies is debugging your Network Policies. Complex applications typically communicate with many endpoints over the network and it might be sometimes time-consuming & difficult to pinpoint which missing or misconfigured Network Policy is the culprit when application runs into errors. In such a case, application logs should provide some insights. However, depending on the quality of application logs or availability of log searching tools, just logs might not be sufficient. In such a case networking tools are likely to provide the most help. 117 | 118 | Running a tool like tcpdump on Kubernetes nodes where the target application pods are running, might come in very handy. You can run tcpdump as a sidecar to your application pods or run tcpdump on a Kubernetes node's interface and apply the necessary filters. [This post](https://itnext.io/generating-kubernetes-network-policies-by-sniffing-network-traffic-6d5135fe77db) and [this one](https://xxradar.medium.com/how-to-tcpdump-effectively-in-kubernetes-part-1-a1546b683d2f) should give you an idea about how it can be done. 119 | 120 | Moreover, depending on the CNI tool that you use, you might be able to extract useful information from CNI software directly. For NSX-T users, [this](https://blogs.vmware.com/management/2019/06/kubernetes-insights-using-vrealize-network-insight-part-1.html) might come in handy, and for Cilium there are some nice tools too, see [here](https://docs.cilium.io/en/v1.9/policy/troubleshooting/#policy-tracing) and [here](https://github.com/cilium/hubble). For Antrea, you might want to checkout [this](https://github.com/vmware-tanzu/antrea/blob/main/docs/network-flow-visibility.md) and for iptables relevant issues [this](https://github.com/box/kube-iptables-tailer). 121 | -------------------------------------------------------------------------------- /network-policies/cilium-editor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/network-policies/cilium-editor.png -------------------------------------------------------------------------------- /network-policies/network_perimeter_kubernetes.drawio: -------------------------------------------------------------------------------- 1 | 7Vzdc6M2EP9r/HAPYZBAAj9e7KRfc22muZlenzKckbEajHyAz07/+kogzIdkg9OA7fgykxgtYoV+u9pdrdYZWZPl9qfYWy0+MZ+EI2j625E1HUEIoAP5h6C85BQMxzkhiKkvO5WER/ovkURTUtfUJ0mtY8pYmNJVnThjUURmaY3mxTHb1LvNWVgfdeUFRCE8zrxQpf5F/XSRU13olPSfCQ0WxcgAy/ktvaKznEmy8Hy2qZCsu5E1iRlL86vldkJCAV6BS/7c/Z67uxeLSZR2eeDL4z837Nfp7zdkHdyBz38Ez/NPN1bO5bsXruWEfyfphsXP8p3TlwIIzoxjzhu3fCIrQZyFbM15324WNCWPK28miBuuBpy2SJchbwF+KUcgcUq2e18d7ADhmkTYkqTxC+8iH0CFNkglAg40UE7ZlEIBY9lrUROIJHpSEYId9xIrfiHhOgI6W4FOwSxm68gngok5CE52AyZkKiBBVwMS7A0kpx2kgKO02jt5ua69r0V381hQLMeuo+KqqOhAASbqCRTYDgpdZnapqh+SNKXLgI8a0q/874yvyScvTvlllK9bGom79zQmGy8Mn0z4BKC75b/GStzpoGL75ahC3AKh25daYQXBz7Hn05SyyBMPF/NPFGD5zNM6rkkas2cyYSGLOSVikcB9TsOwQfJCGkTC7HGkCKffChwp9xQf5Y0l9X0xjHah103BmwtCcoF1A+CqRtLWLf++xAQ6aPpRNrKQQZy//Jv4lqbNVDGDWGce7L7MA0DtqHnJKo925nQrwKuCtGI0SrOXQrcjNNWpbmFe5ixKZczFZaW3Mctk5hH++dv6K4m5lSGJkXwPenFYFlBNM9Jgb7t9WWagQF9OW3ijcJ0IBC/YqrQ4y4ZAsCoQAIc0IVAToXpLkmQ4vFsxANyQg6nKYVBLDt12m9R3INfABGi8m24HMO7LTo/bIaloou8li7qbS+pKVNXKEbSg66A7qKgwvzPPfhRDf2sapghZJ6aBLfEhbD+cZDdARoYN6lhLzVg0e473MHayp/l9DRPQoAkdqvXNfFOxm1xuA5FBMJ5LN0O5zohpxr9kF9MV818fOOUafDCE1WmP09uKMhX1eSjmd5km7SjYi4AV1da0c+KA1VL9vyKRQmHnIdl+FBkmjg+JfHk5nYVektBZXWZkS9MvAkvDdcey/XfWxi6W7elWgp01XiqNBxJTPj8hxIzWsBT3965pllIivpLRalhYPh22jmekzbYdXCy6uKygxST0Uvq9/ho6YckRHoQhq0SFzh7vV7DIX18+BSsprwYjy60zgpZVZ5R6cUBShVGmO7tp/49IvkOOqG+vCZu5M9fR5M4cnePsbVuoxnQ9wHJAIgcVWweFAwwTQWwB5Lhj27GcnpDpEGUpO79VxTrIJG3FYOh2zzuPsEtJm80ARcQmNAo+M+4LplZlx6jbIXozsdiffBrzN2MChvu5TMAcs1U8oCsHBWbzwMcChSepig0jA8PetumqA/8R//Ua/3HqioVvr0uFpcRGIxwByChWPHZtbkmxGp5YhqPRPBsZu4dyw9GXFqoRy+4Ex3xgIZ1R8v6yoPvMeMHGMkAzstRb8IokLY0YgWuMa3LsK+7s4BEvP+48WTxpNcNArgxgXP400oldw0uk8h00vLR0GQgcipXN3TIO8gx5ThCZ3po+4W9rVty4SbIc8EfewcKrbXmz4AIKNsK1Z5zq3Dm5MmJDccuT47bsfjO0qcUnGmPUyYAcl2kD7li18boUdG/HyjpT0ItQ4bsVqq0IFWv2PMOKVd0KXkKlhY3OodKiw4HYiUstdi7mdKUWtnoorqCU76oba5CJvV4qJoIORmVvX4pha46XXKxRrd5KMewOBSo91WJojZ82DO8qoNawOVeR8ynjsNV8x7XUcewTxXkWctgdzrrOrJDDtk5eyIE6ZIjeZyEHGquGfdBCDnTthRwNgWBbFciwhRwIKhK5wkIOpCk+HdSSo0HOXo4q5LDxiQs5UIdjuh+Z/LOp5EDtR0KDVnIgdXv6His59sF+npUcqMNu+Hwz6lwS8Us+jNj7SkI+Dtq1y3Gy1ku11Rwpn/yb1IYU5jLPbh/aNbQftg9YRYKa29jXVpEgMGwVCTqDL9E0q0hsfPIqEjRISeoBiZxtFQnqsFu/sjIS1F79eaIykiIY/hF8XkoZyT5duuQyEnyVZST77PillpHgwc6Or6cgwC6+0H+qk2PcIc6p7FokEjs/AeqQXmoND2ycHL62JNw1Wxj1HMw7qrvPQtGFF3HoBUzf1iQR7lgsvt3qjXcL70PDLr9ou8EPo/L0jF9OCv5/5vzP1JTX1vcbLGZ7XF/MluUqixnojjxt2NNqdlRH+8gj5LAi+0z0qtx5B1X0VyFFZNaliDXfhgQ62/OKlB9vlv+SJV/05T+2se7+Aw== -------------------------------------------------------------------------------- /network-policies/network_perimeter_kubernetes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/network-policies/network_perimeter_kubernetes.png -------------------------------------------------------------------------------- /opa-policies/conformance-testing/README.MD: -------------------------------------------------------------------------------- 1 | --- 2 | title: Enforcing policies in Kubernetes 3 | tags: ['kubernetes', 'opa', 'rego', 'policy', 'policy enforcement'] 4 | status: draft 5 | --- 6 | # Enforcing policies in Kubernetes 7 | 8 | Kubernetes(K8S) has established itself as the go-to platform for container based workloads and many companies have either already started or going to start soon migrating their workloads onto Kubernetes. 9 | 10 | Kubernetes offers so many capabilities out of box, and it exposes many infrastructure related controls to developers. Developers, who are not used to dealing with those infrastructure level concerns, might struggle to grasp all those new controls and abstractions which are not at their disposal. Trainings(such as ones on [here](https://kube.academy/)) will certainly help to bring developers up to speed with K8S but if we want to really ensure that our applications fulfill all the requirements (such as availability, security, etc.) we want them fulfill, we better do a bit more than just publishing best practices in an intranet wiki page and expect developers to adhere to those best-practices religiously. 11 | 12 | But how can you enforce your company's K8S best practices? How can you create 'policies' that covers security and high availability concerns? How can you govern the behavior of a software service? 13 | 14 | *All the relevant files can be found in [this git repo](https://github.com/mcelep/blog/tree/master/opa-policies/conformance-testing).* 15 | ## Enter Open Policy Agent(OPA) 16 | 17 | [Open Policy Agent(OPA)](https://www.openpolicyagent.org/) is a Cloud Native Computing Foundation project and it aims at solving 'policy enforcement' problem across the Cloud-Native stack(Kubernetes, Docker, Envoy, Terraform,etc.). 18 | 19 | OPA comes with a language called [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) to author policies efficiently and other than Rego, there are a number of tools & components in OPA to make use of policies easier & efficient. 20 | 21 | ## Different options to enform policies & run policy checks 22 | 23 | We see several options to enforce policies and inform users about conformance to policies. 24 | 25 | ### Admission Controllers 26 | 27 | The ultimate level of enforcing policies happens directly on a K8S cluster. That is, one can hook into [Admission Controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) mechanism of Kubernetes to reject K8S Resource modifications that are not allowed by policies. 28 | 29 | OPA comes with a component called [Gatekeeper](https://www.openpolicyagent.org/docs/latest/kubernetes-introduction/) This components is an Admission Controller, and it aims at managing and enforcing policies easily on K8S clusters. 30 | 31 | OPA Gatekeeper can (as of 30.10.2020) either deny a admission request (optionally providing some explanation about why an admission was refused) or accept an admission request with no explanation. At the moment there's not a way to accept an admission but provide a **warning** back to user(an example use case could look like: a request might be admitted at that point in time but will not be admitted any more at a later date due to upcoming policy updates). There is [an open issue](https://github.com/open-policy-agent/gatekeeper/issues/701) on Github that's for adding a *warn* action. 32 | 33 | Using *Admission Controllers* brings a binary approach to accepting changes on API resources on a K8S cluster. This helps to make sure no non-policy-conformal workload runs on the platform but it misses to differentiate between recommended vs. obligatory practices. This is where a 'report card' based approach comes handy. 34 | 35 | ### Visualization / Report cards 36 | 37 | Creating a 'conformance' report for workloads running on K8S clusters is a good way to increase awareness among application teams about how their applications fulfill the requirements. In our professional services engagements which includes application commissioning into production, we typically prepare a custom "Production Readiness Checklist" document that includes both obligatory as well as recommended items. There might be certain points in such a list that pretty much universal but some are specific to customers due to needs such as high availability, performance or regulatory requirements (e.g. Financial Service regulations). 38 | 39 | Creating a report card that is generated programmatically upon demand (or periodically) and showing this report to users in a visually pleasing way could be very helpful in many large organizations that has many K8S applications. 40 | 41 | [Polaris OS project](https://github.com/FairwindsOps/polaris) comes with such a feature out of the box, see the image below: 42 | 43 | ![report](https://github.com/mcelep/blog/blob/master/opa-policies/conformance-testing/dashboard-screenshot.png?raw=true) 44 | 45 | When creating such a report, displaying not only failed policy checks but also providing some explanations about why such a policy rule was created and how to fix that issue (typically by providing a link to an internal wiki page), will be much appreciated by readers of the report. 46 | 47 | ### CI/CD 48 | 49 | Another point/stage where you could test K8S resources against policies is CI/CD pipelines and to be more precise it's the CD(Continuous Delivery) part of CI/CD. It should be quite east to add a 'policy check step' to deployment pipelines in most of the CI/CD solutions out there. And the benefit of running policy conformance checks as part of a deployment pipeline is that, conformance check results can include not only **Ok**s or **Nok**s but also warnings. A CI/CD system might also allow you to save reports Moreover, if K8S clusters(especially production clusters) can only be accessed via CD pipelines (and not directly by users as we see in many enterprises) and if you can enforce such a 'policy check step' in all deployment pipelines, you end up with a solution that is similar to an 'Admission Controller'. 50 | 51 | ## OPA policies in practice 52 | 53 | In this section, we will write a couple of policies to get a better understanding about what the process feels like. 54 | 55 | ### Rego 56 | 57 | [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) is a DSL that is not that difficult to get into but that said it takes some getting used to. 58 | 59 | [This](https://academy.styra.com/courses/opa-rego) free course can be helpful if you want to start with a self-paced video based training. 60 | 61 | Best way to learn it is use it and while learning Rego, practicing in an interactive shell can be very helpful, here are some options: 62 | 63 | - [Playground](https://play.openpolicyagent.org/) comes in handy for testing out rego expressions online. 64 | 65 | - [REPL environment for rego](https://github.com/fugue/fregot) can be used if you want to have a local environment to test things out in rego. 66 | 67 | If you use Visual Studio Code as your editor, you might also find [this](https://marketplace.visualstudio.com/items?itemName=tsandall.opa) plugin helpful when developing Rego policies. 68 | 69 | ### Conftest 70 | 71 | [Conftest](https://www.conftest.dev/) is a utility to help writing/maintaining [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) based policies against files(structured data). 72 | 73 | We will be using conftest to run unit tests. Both OPA cli as well as conftest has means to run unit tests. As the policy files we create become more complex, the unit tests will give us some peace of mind. When writing those unit tests, we should pay attention to testing both positive and negative cases. For example if we're testing a rule that should 'warn' us, we need to write unit tests that would trigger a warn but also one that is the 'happy path' as in there's no warning triggered. 74 | 75 | The policies that we aim to write cover the following concerns for deployments: 76 | 77 | - A deployment should have at least 2 replicas (for high availability) 78 | - Containers in a deployment should have a liveness probe 79 | - Containers is a deployment should have a readiness probe 80 | - Containers in a deployment should define resources for both *limits* and *requests* 81 | - Deployment should have anti-affinity rules setup 82 | 83 | These concerns above are meant to just provide a basic set of best practices for running apps in production on a K8S cluster. They are not meant to be complete and they are just used as input for experimenting with Rego policies. 84 | 85 | Under *policy* folder there are policy files and their corresponding unit test files(with suffix: *_test*). 86 | 87 | Conftest provides 3 levels for rules: *deny, violation, and warn*. In our example policy file for deployments(Deployment.rego), we decided to use *warn* level and also included a descriptive name in the part that follows *warn_*. for example ```warn_at_least_2_replicas[msg]```. The variable which is called *msg* in this example, is used as an output variable. In our rules, the value of *msg* is static, it does not depend on data this rule receives but you might want to make the value of *msg* dynamic, to provide explanation/hints about what part of input data fails to comply with the rule. 88 | 89 | There is an accompanying unit test *deployment_test.rego* provided in the same folder as the policy file *deployment.rego*. Data used in unit tests is also embedded into the unit test file. For example, for tests which target rule *warn_at_least_2_replicas* , there are 2 variables called *deployment_1_replica* and *deployment_2_replica* which contain the relevant data. Embedding large amount test data into the unit test file itself is not an ideal solution though, that part should be improved. In the *helpers* there is a python script which was written to convert yaml into json while removing empty elements as they proved to be problematic for the rego compiler. 90 | 91 | #### Running unit tests 92 | 93 | Conftest comes with unit testing capability to check sanity of the rules and this proves to be a very useful feature. Rego, although it's not a very difficult language to grasp, takes some getting used to so writing unit tests for both happy & unhappy paths for each rule will contribute to improved quality assurance. 94 | 95 | Unit tests can be executed with the following command: 96 | ```bash 97 | #Make sure you are in the folder that includes the folder policy 98 | conftest verify --trace 99 | ``` 100 | 101 | 102 | #### Executing policy checks 103 | 104 | The policy files can be tested against static files: 105 | ```bash 106 | #Make sure you are in folder conftest 107 | cat helpers/deployment.yaml | conftest test - 108 | ``` 109 | 110 | or live Kubernetes resources: 111 | ```bash 112 | k get deployments nginx -n test -o yaml | conftest test - 113 | + kubectl get deployments nginx -n test -o yaml 114 | WARN - There must be a readinessProbe for each container 115 | WARN - There must be at least 2 replicas for each deployment 116 | WARN - Pod affinity rules should have been set 117 | 118 | 8 tests, 5 passed, 3 warnings, 0 failures, 0 exceptions 119 | ``` 120 | -------------------------------------------------------------------------------- /opa-policies/conformance-testing/dashboard-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/opa-policies/conformance-testing/dashboard-screenshot.png -------------------------------------------------------------------------------- /opa-policies/conformance-testing/helpers/convert_to_python_map.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import yaml 3 | import sys 4 | import json 5 | 6 | 7 | def clean_dict(d): 8 | for key, value in list(d.items()): 9 | if value is None or (isinstance(value, dict) and len(value) == 0): 10 | del d[key] 11 | elif isinstance(value, dict): 12 | clean_dict(value) 13 | return d # For convenience 14 | 15 | 16 | def process_file(inputfile: str): 17 | if inputfile.endswith('.yaml'): 18 | with open(inputfile) as yaml_file: 19 | map = yaml.safe_load(yaml_file) 20 | map_as_str = str(clean_dict(map)) 21 | print(map_as_str.replace('\'', '"')) 22 | else: 23 | raise Exception("File should have a suffix of .yaml") 24 | 25 | 26 | def main(argv): 27 | inputfile = '' 28 | try: 29 | inputfile = argv[0] 30 | except: 31 | print('convert_to_python_map ') 32 | sys.exit(2) 33 | process_file(inputfile) 34 | 35 | 36 | if __name__ == '__main__': 37 | main(sys.argv[1:]) 38 | -------------------------------------------------------------------------------- /opa-policies/conformance-testing/helpers/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: nginx 7 | name: nginx 8 | spec: 9 | progressDeadlineSeconds: 600 10 | replicas: 2 11 | revisionHistoryLimit: 10 12 | selector: 13 | matchLabels: 14 | app: nginx 15 | strategy: 16 | rollingUpdate: 17 | maxSurge: 25% 18 | maxUnavailable: 25% 19 | type: RollingUpdate 20 | template: 21 | metadata: 22 | creationTimestamp: 23 | labels: 24 | app: nginx 25 | spec: 26 | affinity: 27 | podAntiAffinity: 28 | requiredDuringSchedulingIgnoredDuringExecution: 29 | - labelSelector: 30 | matchExpressions: 31 | - key: app 32 | operator: In 33 | values: 34 | - nginx 35 | topologyKey: kubernetes.io/hostname 36 | containers: 37 | - image: nginxinc/nginx-unprivileged 38 | imagePullPolicy: Always 39 | livenessProbe: 40 | failureThreshold: 3 41 | httpGet: 42 | path: "/" 43 | port: 8080 44 | scheme: HTTP 45 | periodSeconds: 10 46 | successThreshold: 1 47 | timeoutSeconds: 1 48 | readinessProbe: 49 | failureThreshold: 3 50 | httpGet: 51 | path: "/" 52 | port: 8080 53 | scheme: HTTP 54 | periodSeconds: 10 55 | successThreshold: 1 56 | timeoutSeconds: 1 57 | resources: 58 | limits: 59 | cpu: 1 60 | memory: 256Mi 61 | requests: 62 | cpu: 1 63 | memory: 256Mi 64 | dnsPolicy: ClusterFirst 65 | restartPolicy: Always 66 | schedulerName: default-scheduler 67 | securityContext: {} 68 | terminationGracePeriodSeconds: 30 69 | -------------------------------------------------------------------------------- /opa-policies/conformance-testing/helpers/requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml -------------------------------------------------------------------------------- /opa-policies/conformance-testing/policy/deployment.rego: -------------------------------------------------------------------------------- 1 | package deployment 2 | 3 | warn_at_least_2_replicas[msg] { 4 | input.kind == "Deployment" 5 | 2>input.spec.replicas 6 | msg := "There must be at least 2 replicas of a deployment" 7 | } 8 | 9 | warn_readines_probe[msg] { 10 | input.kind == "Deployment" 11 | containers = input.spec.template.spec.containers[_] 12 | not has_key(containers,"readinessProbe") 13 | msg := "There must be a readinessProbe for each container" 14 | } 15 | 16 | warn_liveness_probe[msg] { 17 | input.kind == "Deployment" 18 | containers = input.spec.template.spec.containers[_] 19 | not has_key(containers,"livenessProbe") 20 | msg := "There must be a livenessProbe for each container" 21 | } 22 | 23 | warn_resources[msg] { 24 | input.kind == "Deployment" 25 | containers = input.spec.template.spec.containers[_] 26 | not has_key(containers,"resources") 27 | msg := "There should be a resources element defined for each container" 28 | } 29 | 30 | warn_resource_limits[msg] { 31 | input.kind == "Deployment" 32 | containers = input.spec.template.spec.containers[_] 33 | resource = containers["resources"] 34 | not has_key(resource,"limits") 35 | msg := "There should be a limits element set for resources for each container" 36 | } 37 | 38 | warn_resource_requests[msg] { 39 | input.kind == "Deployment" 40 | containers = input.spec.template.spec.containers[_] 41 | resource = containers["resources"] 42 | not has_key(resource,"requests") 43 | msg := "There should be a requests element set for resources for each container" 44 | } 45 | 46 | warn_affinity["Pod affinity rules should have been set"]{ 47 | input.kind == "Deployment" 48 | pod = input.spec.template.spec 49 | not has_key(pod,"affinity") 50 | } 51 | 52 | warn_affinity_pod_anti_affinity["Pod podAntiAffinity rules should have been set"]{ 53 | input.kind == "Deployment" 54 | affinity = input.spec.template.spec.affinity 55 | not has_key(affinity,"podAntiAffinity") 56 | } 57 | 58 | # This function is from https://blog.kenev.net/posts/check-if-key-exists-in-object-in-rego-42pp 59 | has_key(x, k) { _ = x[k] } 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /opa-policies/conformance-testing/policy/deployment_test.rego: -------------------------------------------------------------------------------- 1 | package deployment 2 | 3 | deployment_1_replica := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 1, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"requests": {"cpu": 1, "memory": "256Mi"}}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 4 | 5 | deployment_2_replica := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"requests": {"cpu": 1, "memory": "256Mi"}}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 6 | 7 | test_1_replica_warned { 8 | warn_at_least_2_replicas with input as deployment_1_replica 9 | } 10 | 11 | test_2_replica_allowed { 12 | not warn_at_least_2_replicas["There must be at least 2 replicas of a deployment"] with input as deployment_2_replica 13 | } 14 | 15 | deployment_with_readiness := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 16 | 17 | deployment_without_readiness_liveness := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always"}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 18 | 19 | deployment_with_liveness := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 20 | 21 | test_readiness_warned { 22 | warn_readines_probe with input as deployment_without_readiness_liveness 23 | } 24 | 25 | test_readiness_probe_allowed { 26 | not warn_readines_probe["There must be a readinessProbe for each container"] with input as deployment_with_readiness 27 | } 28 | 29 | test_liveness_allowed { 30 | not warn_liveness_probe["There must be a livenessProbe for each container"] with input as deployment_with_readiness 31 | } 32 | 33 | test_liveness_warned { 34 | warn_liveness_probe with input as deployment_without_readiness_liveness 35 | } 36 | 37 | deployment_with_resources := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"limits": {"cpu": 1, "memory": "256Mi"}, "requests": {"cpu": 1, "memory": "256Mi"}}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 38 | 39 | deployment_without_resources := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 40 | 41 | deployment_without_resource_limits := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"requests": {"cpu": 1, "memory": "256Mi"}}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 42 | 43 | deployment_without_resource_requests := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"limits": {"cpu": 1, "memory": "256Mi"}}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 44 | 45 | test_resources_allowed { 46 | not warn_resources["There should be a resources element defined for each container"] with input as deployment_with_resources 47 | } 48 | 49 | test_resources_warned { 50 | warn_resources["There should be a resources element defined for each container"] with input as deployment_without_resources 51 | } 52 | 53 | test_resources_limits_warned { 54 | warn_resource_limits with input as deployment_without_resource_limits 55 | } 56 | 57 | test_resources_requests_warned { 58 | warn_resource_limits with input as deployment_without_resource_requests 59 | } 60 | 61 | deployment_with_anti_affinity := {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx"}, "name": "nginx"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 2, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "nginx"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"labels": {"app": "nginx"}}, "spec": {"affinity": {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{"labelSelector": {"matchExpressions": [{"key": "app", "operator": "In", "values": ["nginx"]}]}, "topologyKey": "kubernetes.io/hostname"}]}}, "containers": [{"image": "nginxinc/nginx-unprivileged", "imagePullPolicy": "Always", "livenessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "readinessProbe": {"failureThreshold": 3, "httpGet": {"path": "/", "port": 8080, "scheme": "HTTP"}, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 1}, "resources": {"limits": {"cpu": 1, "memory": "256Mi"}, "requests": {"cpu": 1, "memory": "256Mi"}}}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "terminationGracePeriodSeconds": 30}}}} 62 | 63 | test_with_anti_affinity_allowed { 64 | not warn_affinity["Pod affinity rules should have been set"] with input as deployment_with_anti_affinity 65 | not warn_affinity_pod_anti_affinity["Pod podAntiAffinity rules should have been set"] with input as deployment_with_anti_affinity 66 | } 67 | -------------------------------------------------------------------------------- /prometheus-advanced/README.MD: -------------------------------------------------------------------------------- 1 | --- 2 | title: Prometheus configuration with custom alert labels for platform and application level alerts 3 | tags: ['prometheus', 'kubernetes', 'cloud-native', 'monitoring', 'alert-routing', 'alerting','observability'] 4 | status: draft 5 | --- 6 | 7 | # Prometheus configuration with custom alert labels for platform and application level alerts 8 | 9 | Recently, we've (me and my teammate Markus) worked with a customer on a Kubernetes monitoring solution where we had to fulfill the following requirements: 10 | 11 | 1. Deploy [Prometheus](https://prometheus.io) with easy life-cycling in mind. What I mean by easy life-cycling is upgrading Prometheus instances with minimal effort and have reproducible deployments 12 | 1. Have an initial set of alerting rules in place to cover basic platform & app monitoring needs 13 | 1. Forward alerts to [IBM's Tivoli Netcool/OMNIbus](https://www.ibm.com/support/knowledgecenter/SSSHTQ/landingpage/NetcoolOMNIbus.html) with a label that uniquely identifies the source of the alert 14 | 15 | In this article, we will talk about how we came up with a design to fulfill these requirements above. 16 | We will not cover Prometheus fundamental knowledge in this article, there are enough sources on the web to learn the basics by yourself. 17 | 18 | All the files that are referred to in this article can be found [here](https://github.com/mcelep/blog/tree/master/prometheus-advanced). 19 | 20 | 21 | ## Helm Chart: kube-prometheus-stack 22 | For taking care of the first two requirements,[kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/a3e6d68b12284f26d216250b57b2240cc9519224/charts/kube-prometheus-stack) can be used. This helm chart was formerly named [prometheus-operator](https://github.com/helm/charts/tree/b9278fa98cef543f5473eb55160eaf45833bc74e/stable/prometheus-operator). You can find the details of the kube-prometheus (which includes the Prometheus operator,as well as other useful bits) [here](https://github.com/prometheus-operator/kube-prometheus). 23 | 24 | [Helm charts](https://helm.sh/docs/topics/charts/) are one of the most common ways to deploy software on Kubernetes these days and having an [operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator) for software also comes in handy for operations such as upgrading software version,backup/restore and many other actions that would be typically triggered by administrator of software manually on day 2. 25 | 26 | ### Prometheus rulez :) 27 | The other interesting thing that comes with **kube-prometheus-stack** helm chart is a set of rules that covers most common needs for Kubernetes platforms. 28 | The rules are listed [here](https://github.com/prometheus-community/helm-charts/tree/a3e6d68b12284f26d216250b57b2240cc9519224/charts/kube-prometheus-stack/templates/prometheus/rules-1.14) and Prometheus rules are simply wrapped inside a [custom resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) with the following kind: 29 | ``` 30 | apiVersion: monitoring.coreos.com/v1 31 | kind: PrometheusRule 32 | ``` 33 | 34 | One should bear in mind that, although these rules are good to start with, typically a team responsible for managing K8S platforms will most probably decide to tweak those rules, add new ones, remove some of them over time as they see fit. 35 | 36 | Rules listed [here](https://github.com/prometheus-community/helm-charts/tree/a3e6d68b12284f26d216250b57b2240cc9519224/charts/kube-prometheus-stack/templates/prometheus/rules-1.14) cover two main themes for alerts. Most of rules are for monitoring a Kubernetes platform itself and meant to be taken care by operators of the platform. However, there are also rules concerning workloads running on K8s, e.g. [kubernetes-apps.yaml](https://github.com/prometheus-community/helm-charts/blob/a3e6d68b12284f26d216250b57b2240cc9519224/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml). Alerts that are triggered due to platform events should be routed differently than workload alerts and this is why we need to differentiate between platform/system vs. workload. In other words, different individuals/team should be alerted based on Prometheus rules. In the rest of this post, we will go into details of how Prometheus was configured to always include a custom label with the right value i.e. Platform ID or Workload ID. 37 | 38 | 39 | ## Solution Design 40 | 41 | Image below captures the overall system design. 42 | 43 | ![System overall design](https://github.com/mcelep/blog/blob/master/prometheus-advanced/system-overall.png?raw=true) 44 | 45 | Overall goal is to differentiate between platform level alerts vs alerts that would originate from workload that runs on the cluster i.e. application alerts. Team(s) that are responsible for operating K8S clusters are typically different than teams which run workloads on those clusters. So for making sure that alerts can be routed to the right teams, we need some identifiers that can be used on Tivoli Netcool/OMNIbus side to make the right routing call. We will not go into details how Tivoli Netcool/OMNIbus can be configured to route alerts to different teams. 46 | 47 | As Tivoli Netcool/OMNIbus can receive http calls, we will be using webhooks to send alerts from [Prometheus/Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) to Tivoli Netcool/OMNIbus. The only thing that will be required for making the right routing on Tivoli side is a label that we will send as a part of alert data. The key for this label is decided as **example.com/ci_monitoring**. 48 | 49 | ### Rules Configuration 50 | [This file](./prometheus-operator/values.yml.j2) includes the custom configuration we came up with for the Prometheus Operator deployment. It's a jinja template file, and the reason for that is we replace certain values in that template depending on target K8S cluster such as cluster-name, container image registry hostname, alert sending destinations (email, webhook). All helm charts come with a default *values.yml* file and our jinja template is based on [this file](https://github.com/prometheus-community/helm-charts/blob/a3e6d68b12284f26d216250b57b2240cc9519224/charts/kube-prometheus-stack/values.yaml). 51 | 52 | 53 | Prometheus operator comes with a set of Alerting rules that are [enabled by default)(https://github.com/prometheus-community/helm-charts/blob/564086af6157d2da6a9a5f086010f1cb3a93babd/charts/kube-prometheus-stack/values.yaml#L30) and due to our need of overwriting [kubernetesApps rules](https://github.com/prometheus-community/helm-charts/blob/a3e6d68b12284f26d216250b57b2240cc9519224/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml), the following section in values file is included to disable creation of **KubernetesApps** rules: 54 | ```yaml 55 | defaultRules: 56 | create: true 57 | rules: 58 | kubernetesApps: false 59 | ``` 60 | 61 | After that,we start adding our own version of the *kubernetesApps* rules, and the following is an example excerpt: 62 | ```yaml 63 | ## Provide custom recording or alerting rules to be deployed into the cluster. 64 | additionalPrometheusRules: 65 | - name: kubernetes-apps 66 | groups: 67 | - name: kubernetes-apps 68 | rules: 69 | - alert: KubePodCrashLooping-System 70 | annotations: 71 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container 72 | }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. 73 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping 74 | expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", 75 | namespace=~"[% system_namespaces_regex %]"}[15m]) * 60 * 5 > 0 76 | for: 15m 77 | labels: 78 | severity: critical 79 | label_example_com_ci_monitoring: [% ci_cluster_id %] 80 | - alert: KubePodCrashLooping 81 | annotations: 82 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container 83 | }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. 84 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping 85 | expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", 86 | namespace!~"[% system_namespaces_regex %]"}[15m]) * 60 * 5 > 0 87 | for: 15m 88 | labels: 89 | severity: critical 90 | ``` 91 | The rules are based on the original [kubernetes-apps.yaml](https://github.com/helm/charts/blob/b9278fa98cef543f5473eb55160eaf45833bc74e/stable/prometheus-operator/templates/prometheus/rules-1.14/kubernetes-apps.yaml) and we add two versions for each rule. One for application namespaces and one for system namespaces. 92 | For controlling what a system namespace is we use a jinja placeholder: ```[% system_namespaces_regex %]``` and the value we've used for this placeholder is: ```default|kube-system|pks-system|monitoring```. (see the file under ```vars/test.yaml`` for all values that are overwritten) 93 | 94 | We add labels to Prometheus alerts that are sent from AlertManager to Tivoli side and we make sure that alert queries that are relevant for applications always include that label. In our configuration, this label is called *label_example_com_ci_monitoring*. 95 | 96 | For system relevant alerts, we include a label to represent Cluster ID which is again controlled by a jinja placeholder ```[% ci_cluster_id %]```. Here is the relevant section: 97 | ```yaml 98 | - alert: KubePodCrashLooping-System 99 | annotations: 100 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container 101 | }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. 102 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping 103 | expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", 104 | namespace=~"[% system_namespaces_regex %]"}[15m]) * 60 * 5 > 0 105 | for: 15m 106 | labels: 107 | severity: critical 108 | label_example_com_ci_monitoring: [% ci_cluster_id %] 109 | ``` 110 | 111 | For workload alerts, we have a mechanism to automatically populate *label_example_com_ci_monitoring* on the fly based on cluster and namespace. Moreover, if there's an existing label on the metric with key *label_example_com_ci_monitoring*, it will be kept as is. In the next section we explain how this mechanism works. 112 | 113 | ### Additional Alert Relabellings 114 | In helm values file, in a section with path *prometheus.prometheusSpec.externalLabels* , we have some additional alert relabeling that we introduced for putting some extra labels on each alert triggered from this Prometheus instance. We thought K8S cluster ID, which is controlled by a jinja placeholder(``` [% ci_cluster_id %] ```) , could bring a lot of benefit in the future to denote the source of an alert. The key for this label is set as *label_example_com_ci_cluster* and not to be mixed up with *label_example_com_ci_monitoring*. Purpose of this label is to always show the source K8S platform, independent from whether it's a system or namespace/workload alert. The relevant configuration looks like this: 115 | 116 | ```yaml 117 | prometheusSpec: 118 | logLevel: info 119 | externalLabels: 120 | label_example_com_ci_cluster: [% ci_cluster_id %] 121 | ``` 122 | 123 | The following section that is under path: *prometheus.prometheusSpec.additionalAlertRelabelConfigs* controls how we control [relabellings](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs) of alerts sent out from AlertManager: 124 | ```yaml 125 | additionalAlertRelabelConfigs: 126 | - source_labels: 127 | - label_example_com_ci_cluster 128 | action: replace 129 | regex: (.*) 130 | replacement: "[% ci_cluster_id %]" 131 | target_label: __tmp_monitoring 132 | - source_labels: 133 | - namespace 134 | regex: (.+) 135 | replacement: "NS-${1}-[% ci_cluster_id %]" 136 | target_label: __tmp_monitoring 137 | action: replace 138 | - source_labels: 139 | - label_example_com_ci_monitoring 140 | action: replace 141 | regex: (.+) 142 | target_label: __tmp_monitoring 143 | - source_labels: 144 | - __tmp_monitoring 145 | action: replace 146 | regex: (.*) 147 | replacement: "$1" 148 | target_label: label_example_com_ci_monitoring 149 | ``` 150 | 151 | Prometheus relabeling rules are processed in the same order they are in the yaml document and this allows us to: 152 | 1. Populate a temporary variable called *__tmp_monitoring* with a value coming from ```label_example_com_ci_cluster```. 153 | 1. If there is a label with key ```namespace```, we take its value and use it to create our own expression ```"NS-${1}-[% ci_cluster_id %]"```. The ```${1}``` section here captures the namespace value and the second part ```[% ci_cluster_id %]``` is our jinja placeholder. 154 | 1. If there is an existing label called *label_example_com_ci_monitoring* put the value of that label into existing variable: *__tmp_monitoring* 155 | 1. Finally replace label *label_example_com_ci_monitoring*'s with the temporary label value. 156 | 157 | In other words, with this configuration above we always apply a label with the key: *label_example_com_ci_monitoring* and the value of this key is either the original value of key *label_example_com_ci_monitoring* if it's there and if that label does not exist, we generate a value based on namespace and cluster using a pattern: ```NS-${1}-[% ci_cluster_id %]``` if there's a label with *namespace* as its key. If there's not a label with key: *namespace* then we fall back to *ci_cluster_id*. 158 | 159 | In this context, it would be good to remind you that there's a conversion process that takes place when scraping data from Kubernetes into Prometheus. During this conversion a key such as **example.com/ci_monitoring** would be translated in Prometheus into a label called *label_example_com_ci_monitoring*. 160 | 161 | ### Persistency 162 | 163 | Prometheus Operator helm chart comes with persistent volumes disabled by default. In order to enable and configure persistent volumes, the following section is used: 164 | ```yaml 165 | volumeClaimTemplate: 166 | spec: 167 | accessModes: ["ReadWriteOnce"] 168 | resources: 169 | requests: 170 | storage: 50Gi 171 | ``` 172 | 173 | This section is available under path: *prometheus.prometheusSpec.storageSpec*. We have not specified a storage class and we assume that there is a default StorageClass configured for the cluster. Alternatively, in the *spec* section, you can specify a storage class as shown below: 174 | ```yaml 175 | volumeClaimTemplate: 176 | spec: 177 | accessModes: ["ReadWriteOnce"] 178 | storageClassName: XYZ 179 | resources: 180 | requests: 181 | storage: 50Gi 182 | ``` 183 | 184 | ### Alertmanager configuration 185 | 186 | This excerpt below shows how *Prometheus Alertmanager* can be configured to forward alerts: 187 | ```yaml 188 | alertmanager: 189 | enabled: true 190 | 191 | config: 192 | global: {} 193 | receivers: 194 | - name: default-receiver 195 | webhook_configs: 196 | - url: "[% alertmanager_webhook_url %]" 197 | send_resolved: true 198 | email_configs: 199 | - send_resolved: true 200 | to: [% alertmanager_smtp_to %] 201 | from: [% alertmanager_smtp_from %] 202 | smarthost: [% alertmanager_smtp_host %] 203 | require_tls: false 204 | - name: 'null' 205 | route: 206 | group_by: ['job'] 207 | group_wait: 30s 208 | group_interval: 5m 209 | repeat_interval: 24h 210 | receiver: default-receiver 211 | ``` 212 | There are a total of 2 receivers configured in the section above. The first one is called *default-receiver* and this one has **web_hook** and **email** actions that are triggered when an alert is raised. *Webhook* integration allows you send alerts to any http capable endpoint such as IBM Tivoli Netcool/OMNIbus. See [here](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config) for details about configuring webhook listeners. In the *default-receiver* section, email is the second receiver for any points which can be quick way to test things out if you have access to a email server that uses smtp protocol. 213 | During testing purposes, if you want to avoid sending alerts to third parties, you can use the ```null``` receiver. 214 | 215 | ### Dead man's switch 216 | 217 | If prometheus gets deployed on the K8S platform that you want to observe, you risk not getting any alerts because Prometheus could potentially not run if platform does not run properly or at all. In order to fix issue, a common pattern have Prometheus send an alert periodically (*a heartbeat* as it's referred in some other domains) to another monitoring platform that operates in a different fault domain. This pattern is often referred as 'Dead man's switch' and it's controlled by the following section in the [general rules yaml](https://github.com/prometheus-community/helm-charts/blob/b643f58714a063e5a0ddf6832cd00ed29c0f4fab/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/general.rules.yaml): 218 | ```yaml 219 | - alert: Watchdog 220 | annotations: 221 | message: 'This is an alert meant to ensure that the entire alerting pipeline is functional. 222 | This alert is always firing, therefore it should always be firing in Alertmanager 223 | and always fire against a receiver. There are integrations with various notification 224 | mechanisms that send a notification when this alert is not firing. For example the 225 | "DeadMansSnitch" integration in PagerDuty. 226 | ' 227 | expr: vector(1) 228 | labels: 229 | severity: none 230 | ``` 231 | 232 | ## Solution in Action 233 | 234 | ### Tools needed 235 | - Kubernetes CLI 236 | - Helm >= 3.2 237 | - Ansible-playbook 238 | - A K8S cluster 239 | 240 | ### Install Nginx Ingress Controller 241 | 242 | Run the following command to install nginx-ingress: 243 | ``` 244 | ansible-playbook -v playbooks/nginx-ingress.yml -i inventory-local.yaml --extra-vars @vars/test.yml 245 | ``` 246 | 247 | This playbook will talk to the localhost from which you execute the ```ansible-playbook``` binary and it will: 248 | - Add necessary helm repos 249 | - Update helm repos 250 | - Instal Nginx ingress controller which we will later use in the Prometheus installation. 251 | 252 | 253 | ### Install Prometheus 254 | 255 | Run the following command to install and configure Prometheus and its dependencies: 256 | ```yaml 257 | ansible-playbook -v playbooks/prometheus-operator.yml -i inventory-local.yaml --extra-vars @vars/test.yml 258 | ``` 259 | 260 | ### Create a webhook lister 261 | In this section we would like to run a simple webhook listener that prints the details of the http call it receives (including payload) onto stdout. The following commands will run an image that will contain a webhook listener program. 262 | ```yaml 263 | kubectl -n monitoring create deployment webhook --image=mcelep/python-webhook-listener 264 | kubectl expose deployment/webhook --port 8080 265 | ``` 266 | 267 | 268 | ### Create a test app 269 | Run the following commands to create a namespace called *test* and create a deployment called *test* 270 | ```yaml 271 | kubectl create ns test 272 | kubectl -n test create deployment test --image=no-such-image 273 | ``` 274 | 275 | Soon you should have some alerts that are in pending state and there should be after the configured amount of time (and for *KubePodNotReady* it's set to 15 minutes in *values.yml.j2* file) alerts that are firing with the right labels. 276 | 277 | When we check the logs of the webhook listener container, we see that following data was received: 278 | ```json 279 | { 280 | "status": "firing", 281 | "labels": { 282 | "__tmp_monitoring": "NS-test-CL-XYZ", 283 | "alertname": "KubePodNotReady", 284 | "label_example_com_ci_cluster": "CL-XYZ", 285 | "label_example_com_ci_monitoring": "NS-test-CL-XYZ", 286 | "namespace": "test", 287 | "pod": "test-7c8844b6fb-kbj8l", 288 | "prometheus": "monitoring/platform-kube-prometheus-s-prometheus", 289 | "severity": "critical" 290 | }, 291 | "annotations": { 292 | "message": "Pod test/test-7c8844b6fb-kbj8l has been in a non-ready state for longer than 15 minutes.", 293 | "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready" 294 | }, 295 | "startsAt": "2020-10-29T10:49:18.473Z", 296 | "endsAt": "0001-01-01T00:00:00Z", 297 | "generatorURL": "http://prometheus.monitoring.k8s.example.com/graph?g0.expr=sum+by%28namespace%2C+pod%2C+label_example_com_ci_monitoring%29+%28max+by%28namespace%2C+pod%2C+label_example_com_ci_monitoring%29+%28kube_pod_status_phase%7Bjob%3D%22kube-state-metrics%22%2Cnamespace%21~%22default%7Ckube-system%7Cpks-system%7Cmonitoring%22%2Cphase%3D~%22Pending%7CUnknown%22%7D%29+%2A+on%28namespace%2C+pod%29+group_left%28owner_kind%29+max+by%28namespace%2C+pod%2C+owner_kind%2C+label_example_com_ci_monitoring%29+%28kube_pod_owner%7Bowner_kind%21%3D%22Job%22%7D%29%29+%3E+0\\u0026g0.tab=1", 298 | "fingerprint": "0b23d99b9740b21e" 299 | } 300 | ``` 301 | 302 | Namespace *test*, does not have a label with key **example.com/ci_monitoring** so label *label_example_com_ci_monitoring* was auto generated based on our relabelling configuration and thus got the value which is a combination of cluster id and namespace. And that's it! We have our end to end solution in action. -------------------------------------------------------------------------------- /prometheus-advanced/inventory-local.yaml: -------------------------------------------------------------------------------- 1 | hosts: 2 | localhost: 3 | vars: 4 | ansible_connection: local 5 | -------------------------------------------------------------------------------- /prometheus-advanced/nginx-ingress/values.yml: -------------------------------------------------------------------------------- 1 | controller: 2 | name: controller 3 | publishService: 4 | enabled: true 5 | 6 | podSecurityPolicy: 7 | enabled: true 8 | -------------------------------------------------------------------------------- /prometheus-advanced/playbooks/nginx-ingress.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Deploy nginx-ingress with helm 3 | hosts: localhost 4 | gather_facts: no 5 | vars: 6 | helm_chart: bitnami/nginx-ingress-controller 7 | helm_chart_version: 5.6.14 8 | helm_deployment_name: monitoring-ingress 9 | namespace: monitoring 10 | tasks: 11 | - name: helm_repo_add 12 | shell: | 13 | helm repo add bitnami https://charts.bitnami.com/bitnami 14 | - name: helm_repo_update 15 | shell: | 16 | helm repo update 17 | - name: install_deployment 18 | shell: | 19 | helm upgrade {{ helm_deployment_name }} {{ helm_chart }} \ 20 | --version {{ helm_chart_version }} \ 21 | -n {{ namespace }} \ 22 | -f ../nginx-ingress/values.yml \ 23 | --install --create-namespace 24 | -------------------------------------------------------------------------------- /prometheus-advanced/playbooks/prometheus-operator.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Deploy prometheus operator with helm 3 | hosts: localhost 4 | gather_facts: no 5 | vars: 6 | helm_chart: prometheus-community/kube-prometheus-stack 7 | helm_chart_version: 10.1.1 8 | helm_deployment_name: platform 9 | namespace: monitoring 10 | tasks: 11 | - name: render values 12 | template: 13 | src: ../prometheus-operator/values.yml.j2 14 | dest: ../prometheus-operator/values-rendered.yml 15 | variable_start_string: "[%" 16 | variable_end_string: "%]" 17 | - name: helm_repo_add 18 | shell: | 19 | helm repo add prometheus-community https://prometheus-community.github.io/helm-charts && \ 20 | helm repo add stable https://charts.helm.sh/stable 21 | - name: update_helm_repo 22 | shell: | 23 | helm repo update 24 | - name: install_deployment 25 | shell: | 26 | helm upgrade {{ helm_deployment_name }} {{ helm_chart }} \ 27 | --version {{ helm_chart_version }} \ 28 | -n {{ namespace }} \ 29 | -f ../prometheus-operator/values-rendered.yml \ 30 | --install --create-namespace 31 | -------------------------------------------------------------------------------- /prometheus-advanced/prometheus-operator/values.yml.j2: -------------------------------------------------------------------------------- 1 | ## Create default rules for monitoring the cluster 2 | ## 3 | defaultRules: 4 | create: true 5 | rules: 6 | kubernetesApps: false 7 | kubeScheduler: false # kubernetesAbsent rule contains the same content 8 | 9 | ## Provide custom recording or alerting rules to be deployed into the cluster. 10 | additionalPrometheusRules: 11 | - name: kubernetes-apps 12 | groups: 13 | - name: kubernetes-apps 14 | rules: 15 | - alert: KubePodCrashLooping-System 16 | annotations: 17 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container 18 | }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. 19 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping 20 | expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", 21 | namespace=~"[% system_namespaces_regex %]"}[15m]) * 60 * 5 > 0 22 | for: 15m 23 | labels: 24 | severity: critical 25 | label_example_com_ci_monitoring: [% ci_cluster_id %] 26 | - alert: KubePodCrashLooping 27 | annotations: 28 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container 29 | }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes. 30 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping 31 | expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", 32 | namespace!~"[% system_namespaces_regex %]"}[15m]) * 60 * 5 > 0 33 | for: 15m 34 | labels: 35 | severity: critical 36 | - alert: KubePodNotReady-System 37 | annotations: 38 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready 39 | state for longer than 15 minutes. 40 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready 41 | expr: sum by(namespace, pod, label_example_com_ci_monitoring) (max by(namespace, pod,label_example_com_ci_monitoring) (kube_pod_status_phase{job="kube-state-metrics",namespace=~"[% system_namespaces_regex %]",phase=~"Pending|Unknown"}) * on(namespace, pod) group_left (owner_kind) max by(namespace, pod, owner_kind, label_example_com_ci_monitoring) (kube_pod_owner{owner_kind!="Job"})) > 0 42 | for: 15m 43 | labels: 44 | severity: critical 45 | label_example_com_ci_monitoring: [% ci_cluster_id %] 46 | - alert: KubePodNotReady 47 | annotations: 48 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready 49 | state for longer than 15 minutes. 50 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready 51 | expr: sum by(namespace, pod, label_example_com_ci_monitoring) (max by(namespace, pod,label_example_com_ci_monitoring) (kube_pod_status_phase{job="kube-state-metrics",namespace!~"[% system_namespaces_regex %]",phase=~"Pending|Unknown"}) * on(namespace, pod) group_left (owner_kind) max by(namespace, pod, owner_kind, label_example_com_ci_monitoring) (kube_pod_owner{owner_kind!="Job"})) > 0 52 | for: 15m 53 | labels: 54 | severity: critical 55 | - alert: KubeDeploymentGenerationMismatch-System 56 | annotations: 57 | message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment 58 | }} does not match, this indicates that the Deployment has failed but has 59 | not been rolled back. 60 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch 61 | expr: "kube_deployment_status_observed_generation{job=\"kube-state-metrics\"\ 62 | , namespace=~\"[% system_namespaces_regex %]\"}\n !=\nkube_deployment_metadata_generation{job=\"kube-state-metrics\"\ 63 | , namespace=~\"[% system_namespaces_regex %]\"}" 64 | for: 15m 65 | labels: 66 | severity: critical 67 | label_example_com_ci_monitoring: [% ci_cluster_id %] 68 | - alert: KubeDeploymentGenerationMismatch 69 | annotations: 70 | message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment 71 | }} does not match, this indicates that the Deployment has failed but has 72 | not been rolled back. 73 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch 74 | expr: "kube_deployment_status_observed_generation{job=\"kube-state-metrics\"\ 75 | , namespace!~\"[% system_namespaces_regex %]\"}\n !=\nkube_deployment_metadata_generation{job=\"kube-state-metrics\"\ 76 | , namespace!~\"[% system_namespaces_regex %]\"}" 77 | for: 15m 78 | labels: 79 | severity: critical 80 | - alert: KubeDeploymentReplicasMismatch-System 81 | annotations: 82 | message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has 83 | not matched the expected number of replicas for longer than 15 minutes. 84 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch 85 | expr: "kube_deployment_spec_replicas{job=\"kube-state-metrics\", namespace=~\"\ 86 | [% system_namespaces_regex %]\"}\n !=\nkube_deployment_status_replicas_available{job=\"kube-state-metrics\"\ 87 | , namespace=~\"[% system_namespaces_regex %]\"}" 88 | for: 15m 89 | labels: 90 | severity: critical 91 | label_example_com_ci_monitoring: [% ci_cluster_id %] 92 | - alert: KubeDeploymentReplicasMismatch 93 | annotations: 94 | message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has 95 | not matched the expected number of replicas for longer than 15 minutes. 96 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch 97 | expr: "kube_deployment_spec_replicas{job=\"kube-state-metrics\", namespace!~\"\ 98 | [% system_namespaces_regex %]\"}\n !=\nkube_deployment_status_replicas_available{job=\"kube-state-metrics\"\ 99 | , namespace!~\"[% system_namespaces_regex %]\"}" 100 | for: 15m 101 | labels: 102 | severity: critical 103 | - alert: KubeStatefulSetReplicasMismatch-System 104 | annotations: 105 | message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has 106 | not matched the expected number of replicas for longer than 15 minutes. 107 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch 108 | expr: "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\",\ 109 | \ namespace=~\"[% system_namespaces_regex %]\"}\n !=\nkube_statefulset_status_replicas{job=\"kube-state-metrics\"\ 110 | , namespace=~\"[% system_namespaces_regex %]\"}" 111 | for: 15m 112 | labels: 113 | severity: critical 114 | label_example_com_ci_monitoring: [% ci_cluster_id %] 115 | - alert: KubeStatefulSetReplicasMismatch 116 | annotations: 117 | message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has 118 | not matched the expected number of replicas for longer than 15 minutes. 119 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch 120 | expr: "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\",\ 121 | \ namespace!~\"[% system_namespaces_regex %]\"}\n !=\nkube_statefulset_status_replicas{job=\"kube-state-metrics\"\ 122 | , namespace!~\"[% system_namespaces_regex %]\"}" 123 | for: 15m 124 | labels: 125 | severity: critical 126 | - alert: KubeStatefulSetGenerationMismatch-System 127 | annotations: 128 | message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset 129 | }} does not match, this indicates that the StatefulSet has failed but 130 | has not been rolled back. 131 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch 132 | expr: "kube_statefulset_status_observed_generation{job=\"kube-state-metrics\"\ 133 | , namespace=~\"[% system_namespaces_regex %]\"}\n !=\nkube_statefulset_metadata_generation{job=\"\ 134 | kube-state-metrics\", namespace=~\"[% system_namespaces_regex %]\"}" 135 | for: 15m 136 | labels: 137 | severity: critical 138 | label_example_com_ci_monitoring: [% ci_cluster_id %] 139 | - alert: KubeStatefulSetGenerationMismatch 140 | annotations: 141 | message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset 142 | }} does not match, this indicates that the StatefulSet has failed but 143 | has not been rolled back. 144 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch 145 | expr: "kube_statefulset_status_observed_generation{job=\"kube-state-metrics\"\ 146 | , namespace!~\"[% system_namespaces_regex %]\"}\n !=\nkube_statefulset_metadata_generation{job=\"\ 147 | kube-state-metrics\", namespace!~\"[% system_namespaces_regex %]\"}" 148 | for: 15m 149 | labels: 150 | severity: critical 151 | - alert: KubeStatefulSetUpdateNotRolledOut-System 152 | annotations: 153 | message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update 154 | has not been rolled out. 155 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout 156 | expr: "max without (revision) (\n kube_statefulset_status_current_revision{job=\"\ 157 | kube-state-metrics\", namespace=~\"[% system_namespaces_regex %]\"}\n unless\n kube_statefulset_status_update_revision{job=\"\ 158 | kube-state-metrics\", namespace=~\"[% system_namespaces_regex %]\"}\n)\n *\n(\n kube_statefulset_replicas{job=\"\ 159 | kube-state-metrics\", namespace=~\"[% system_namespaces_regex %]\"}\n !=\n kube_statefulset_status_replicas_updated{job=\"\ 160 | kube-state-metrics\", namespace=~\"[% system_namespaces_regex %]\"}\n)" 161 | for: 15m 162 | labels: 163 | severity: critical 164 | label_example_com_ci_monitoring: [% ci_cluster_id %] 165 | - alert: KubeStatefulSetUpdateNotRolledOut 166 | annotations: 167 | message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update 168 | has not been rolled out. 169 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout 170 | expr: "max without (revision) (\n kube_statefulset_status_current_revision{job=\"\ 171 | kube-state-metrics\", namespace!~\"[% system_namespaces_regex %]\"}\n unless\n kube_statefulset_status_update_revision{job=\"\ 172 | kube-state-metrics\", namespace!~\"[% system_namespaces_regex %]\"}\n)\n *\n(\n kube_statefulset_replicas{job=\"\ 173 | kube-state-metrics\", namespace!~\"[% system_namespaces_regex %]\"}\n !=\n kube_statefulset_status_replicas_updated{job=\"\ 174 | kube-state-metrics\", namespace!~\"[% system_namespaces_regex %]\"}\n)" 175 | for: 15m 176 | labels: 177 | severity: critical 178 | - alert: KubeDaemonSetRolloutStuck-System 179 | annotations: 180 | message: Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet 181 | {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready. 182 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck 183 | expr: "kube_daemonset_status_number_ready{job=\"kube-state-metrics\", namespace=~\"\ 184 | [% system_namespaces_regex %]\"}\n /\nkube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"\ 185 | , namespace=~\"[% system_namespaces_regex %]\"} < 1.00" 186 | for: 15m 187 | labels: 188 | severity: critical 189 | label_example_com_ci_monitoring: [% ci_cluster_id %] 190 | - alert: KubeDaemonSetRolloutStuck 191 | annotations: 192 | message: Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet 193 | {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready. 194 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck 195 | expr: "kube_daemonset_status_number_ready{job=\"kube-state-metrics\", namespace=~\"\ 196 | [% system_namespaces_regex %]\"}\n /\nkube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"\ 197 | , namespace=~\"[% system_namespaces_regex %]\"} < 1.00" 198 | for: 15m 199 | labels: 200 | severity: critical 201 | - alert: KubeContainerWaiting-System 202 | annotations: 203 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} 204 | has been in waiting state for longer than 1 hour. 205 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting 206 | expr: sum by(namespace, pod, container,label_example_com_ci_monitoring) (kube_pod_container_status_waiting_reason {job="kube-state-metrics",namespace=~"[% system_namespaces_regex %]"}) > 0 207 | for: 1h 208 | labels: 209 | severity: warning 210 | label_example_com_ci_monitoring: [% ci_cluster_id %] 211 | - alert: KubeContainerWaiting 212 | annotations: 213 | message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} 214 | has been in waiting state for longer than 1 hour. 215 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting 216 | expr: sum by(namespace, pod, container,label_example_com_ci_monitoring) (kube_pod_container_status_waiting_reason {job="kube-state-metrics",namespace!~"[% system_namespaces_regex %]"}) > 0 217 | for: 1h 218 | labels: 219 | severity: warning 220 | - alert: KubeDaemonSetNotScheduled-System 221 | annotations: 222 | message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset 223 | }} are not scheduled.' 224 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled 225 | expr: "kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"\ 226 | , namespace=~\"[% system_namespaces_regex %]\"}\n -\nkube_daemonset_status_current_number_scheduled{job=\"\ 227 | kube-state-metrics\", namespace=~\".*\"} > 0" 228 | for: 10m 229 | labels: 230 | severity: warning 231 | label_example_com_ci_monitoring: [% ci_cluster_id %] 232 | - alert: KubeDaemonSetNotScheduled 233 | annotations: 234 | message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset 235 | }} are not scheduled.' 236 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled 237 | expr: "kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"\ 238 | , namespace!~\"[% system_namespaces_regex %]\"}\n -\nkube_daemonset_status_current_number_scheduled{job=\"\ 239 | kube-state-metrics\", namespace=~\".*\"} > 0" 240 | for: 10m 241 | labels: 242 | severity: warning 243 | - alert: KubeDaemonSetMisScheduled-System 244 | annotations: 245 | message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset 246 | }} are running where they are not supposed to run.' 247 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled 248 | expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", 249 | namespace=~"[% system_namespaces_regex %]"} > 0 250 | for: 10m 251 | labels: 252 | severity: warning 253 | label_example_com_ci_monitoring: [% ci_cluster_id %] 254 | - alert: KubeDaemonSetMisScheduled 255 | annotations: 256 | message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset 257 | }} are running where they are not supposed to run.' 258 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled 259 | expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", 260 | namespace!~"[% system_namespaces_regex %]"} > 0 261 | for: 10m 262 | labels: 263 | severity: warning 264 | - alert: KubeCronJobRunning-System 265 | annotations: 266 | message: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking 267 | more than 1h to complete. 268 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning 269 | expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics", namespace=~"[% system_namespaces_regex %]"} 270 | > 3600 271 | for: 1h 272 | labels: 273 | severity: warning 274 | label_example_com_ci_monitoring: [% ci_cluster_id %] 275 | - alert: KubeCronJobRunning 276 | annotations: 277 | message: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking 278 | more than 1h to complete. 279 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning 280 | expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics", namespace!~"[% system_namespaces_regex %]"} 281 | > 3600 282 | for: 1h 283 | labels: 284 | severity: warning 285 | - alert: KubeJobCompletion-System 286 | annotations: 287 | message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more 288 | than one hour to complete. 289 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion 290 | expr: kube_job_spec_completions{job="kube-state-metrics", namespace=~"[% system_namespaces_regex %]"} 291 | - kube_job_status_succeeded{job="kube-state-metrics", namespace=~"[% system_namespaces_regex %]"} > 292 | 0 293 | for: 1h 294 | labels: 295 | severity: warning 296 | label_example_com_ci_monitoring: [% ci_cluster_id %] 297 | - alert: KubeJobCompletion 298 | annotations: 299 | message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more 300 | than one hour to complete. 301 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion 302 | expr: kube_job_spec_completions{job="kube-state-metrics", namespace!~"[% system_namespaces_regex %]"} 303 | - kube_job_status_succeeded{job="kube-state-metrics", namespace!~"[% system_namespaces_regex %]"} > 304 | 0 305 | for: 1h 306 | labels: 307 | severity: warning 308 | - alert: KubeJobFailed-System 309 | annotations: 310 | message: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. 311 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed 312 | expr: kube_job_failed{job="kube-state-metrics", namespace=~"[% system_namespaces_regex %]"} > 0 313 | for: 15m 314 | labels: 315 | severity: warning 316 | label_example_com_ci_monitoring: [% ci_cluster_id %] 317 | - alert: KubeJobFailed 318 | annotations: 319 | message: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. 320 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed 321 | expr: kube_job_failed{job="kube-state-metrics", namespace!~"[% system_namespaces_regex %]"} > 0 322 | for: 15m 323 | labels: 324 | severity: warning 325 | - alert: KubeHpaReplicasMismatch-System 326 | annotations: 327 | message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the 328 | desired number of replicas for longer than 15 minutes. 329 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch 330 | expr: "(kube_hpa_status_desired_replicas{job=\"kube-state-metrics\", namespace=~\"\ 331 | [% system_namespaces_regex %]\"}\n !=\nkube_hpa_status_current_replicas{job=\"kube-state-metrics\"\ 332 | , namespace=~\"[% system_namespaces_regex %]\"})\n and\nchanges(kube_hpa_status_current_replicas[15m])\ 333 | \ == 0" 334 | for: 15m 335 | labels: 336 | severity: warning 337 | label_example_com_ci_monitoring: [% ci_cluster_id %] 338 | - alert: KubeHpaReplicasMismatch 339 | annotations: 340 | message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the 341 | desired number of replicas for longer than 15 minutes. 342 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch 343 | expr: "(kube_hpa_status_desired_replicas{job=\"kube-state-metrics\", namespace!~\"\ 344 | [% system_namespaces_regex %]\"}\n !=\nkube_hpa_status_current_replicas{job=\"kube-state-metrics\"\ 345 | , namespace!~\"[% system_namespaces_regex %]\"})\n and\nchanges(kube_hpa_status_current_replicas[15m])\ 346 | \ == 0" 347 | for: 15m 348 | labels: 349 | severity: warning 350 | - alert: KubeHpaMaxedOut-System 351 | annotations: 352 | message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running 353 | at max replicas for longer than 15 minutes. 354 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout 355 | expr: "kube_hpa_status_current_replicas{job=\"kube-state-metrics\", namespace=~\"\ 356 | [% system_namespaces_regex %]\"}\n ==\nkube_hpa_spec_max_replicas{job=\"kube-state-metrics\", namespace=~\"\ 357 | [% system_namespaces_regex %]\"}" 358 | for: 15m 359 | labels: 360 | severity: warning 361 | label_example_com_ci_monitoring: [% ci_cluster_id %] 362 | - alert: KubeHpaMaxedOut 363 | annotations: 364 | message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running 365 | at max replicas for longer than 15 minutes. 366 | runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout 367 | expr: "kube_hpa_status_current_replicas{job=\"kube-state-metrics\", namespace!~\"\ 368 | [% system_namespaces_regex %]\"}\n ==\nkube_hpa_spec_max_replicas{job=\"kube-state-metrics\", namespace!~\"\ 369 | [% system_namespaces_regex %]\"}" 370 | for: 15m 371 | labels: 372 | severity: warning 373 | global: 374 | rbac: 375 | create: true 376 | 377 | 378 | alertmanager: 379 | enabled: true 380 | 381 | config: 382 | global: {} 383 | receivers: 384 | - name: default-receiver 385 | webhook_configs: 386 | - url: "[% alertmanager_webhook_url %]" 387 | send_resolved: true 388 | email_configs: 389 | - send_resolved: true 390 | to: [% alertmanager_smtp_to %] 391 | from: [% alertmanager_smtp_from %] 392 | smarthost: [% alertmanager_smtp_host %] 393 | require_tls: false 394 | - name: 'null' 395 | route: 396 | group_by: ['job'] 397 | group_wait: 30s 398 | group_interval: 5m 399 | repeat_interval: 24h 400 | receiver: default-receiver 401 | # routes: 402 | # - match: 403 | # alertname: Watchdog 404 | # receiver: 'null' 405 | 406 | ingress: 407 | enabled: true 408 | annotations: 409 | kubernetes.io/ingress.class: nginx 410 | hosts: 411 | - alertmanager.[% ingress_url_suffix %] 412 | 413 | alertmanagerSpec: 414 | strategy: 415 | type: "Recreate" 416 | 417 | 418 | ## Configuration for kube-state-metrics subchart 419 | ## 420 | kube-state-metrics: 421 | 422 | podSecurityPolicy: 423 | enabled: true 424 | 425 | ## Manages Prometheus and Alertmanager components 426 | ## 427 | prometheusOperator: 428 | enabled: true 429 | 430 | tlsProxy: 431 | enabled: true 432 | resources: {} 433 | 434 | ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). 435 | ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration 436 | ## 437 | namespaces: {} 438 | 439 | ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). 440 | ## 441 | denyNamespaces: [] 442 | 443 | 444 | ## Deploy CRDs used by Prometheus Operator. 445 | ## 446 | createCustomResource: true 447 | 448 | ## Attempt to clean up CRDs created by Prometheus Operator. 449 | ## 450 | cleanupCustomResource: false 451 | 452 | 453 | 454 | ## Deploy a Prometheus instance 455 | ## 456 | prometheus: 457 | 458 | enabled: true 459 | ingress: 460 | enabled: true 461 | annotations: 462 | kubernetes.io/ingress.class: nginx 463 | hosts: 464 | - prometheus.[% ingress_url_suffix %] 465 | 466 | ## Settings affecting prometheusSpec 467 | ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#prometheusspec 468 | ## 469 | prometheusSpec: 470 | logLevel: info 471 | externalLabels: 472 | label_example_com_ci_cluster: [% ci_cluster_id %] 473 | 474 | 475 | ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. 476 | ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. 477 | ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. 478 | ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. 479 | podAntiAffinity: "" 480 | 481 | ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. 482 | ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone 483 | ## 484 | podAntiAffinityTopologyKey: kubernetes.io/hostname 485 | 486 | ## Prometheus StorageSpec for persistent data 487 | ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/storage.md 488 | ## 489 | storageSpec: 490 | volumeClaimTemplate: 491 | spec: 492 | accessModes: ["ReadWriteOnce"] 493 | resources: 494 | requests: 495 | storage: 50Gi 496 | 497 | additionalAlertRelabelConfigs: 498 | - source_labels: 499 | - label_example_com_ci_cluster 500 | action: replace 501 | regex: (.*) 502 | replacement: "[% ci_cluster_id %]" 503 | target_label: __tmp_monitoring 504 | - source_labels: 505 | - namespace 506 | regex: (.+) 507 | replacement: "NS-${1}-[% ci_cluster_id %]" 508 | target_label: __tmp_monitoring 509 | action: replace 510 | - source_labels: 511 | - label_example_com_ci_monitoring 512 | action: replace 513 | regex: (.+) 514 | target_label: __tmp_monitoring 515 | - source_labels: 516 | - __tmp_monitoring 517 | action: replace 518 | regex: (.*) 519 | replacement: "$1" 520 | target_label: label_example_com_ci_monitoring -------------------------------------------------------------------------------- /prometheus-advanced/system-overall.drawio: -------------------------------------------------------------------------------- 1 | 7V1Xd9pMt/41Wes7F/FSL5cIIapAoILg5iyhDmqoIenXnxl6c+zkjZ285zMJtrRnNDPa+9llqr/h7bDqpkbiibFlB98wxKq+4fw3DKNIigC/IKU+UDAEoQ8UN/WtAw29EGS/sY9E5EgtfMvObjLmcRzkfnJLNOMoss38hmakaby7zebEwW2tieHaDwTZNIJH6ty3cu9IpUjiktCzfdc7VY1S7CElNE65j6+SeYYV765IeOcb3k7jOD9chVXbDiD7Tow5PCe8knpuWWpH+XseCJKsqdr5MlSEZkZOOtliTn5Hj8WURlAcX/kbRgWgQM6JQbmg2Xl9ZAa1LeJTwvdsL6oWyIBSSXVJBFcu/C0FRu7EaQgytAI7zU+lgvYdCj5kO7LmXAeWp74RufCO23l+bsuJYcKkHQAYoHl5GIA7FFwGxsoOOMPcuGlcRFY7DuIUJEVxBB92/CA4kb5huCC0wQfQjcB3I0ALbOdS/zUTTwwBbbarK9KRqV07Du08rUGWYypDHp44Qhwnj/LeXeGFONK8a6ggR6JxxKh7LvoiRXBxFOTPCBX750Klnwl1HqebIDasv0SoLPgIwscIFbkRKvlMqORHCdVtnPH/huuesBU7GLmULL/afH+U6V4EohEBE5Q+sHzPPts6svV3cr1DwX+np6Q483M/htw3AbtBS3AOstkHNnR0lyH0LWsPgpO8Hp5oHRPOOSGs5ONboaf7g5dA6bOoH+T6RPqvihpHb2VN4OSjrJlnssY/StbkW/prnsVxUVHc2X8etVbxyzjwwVNjOzeB8wRNmojj/qrI3qu/HwcmBKEolv3/AyaS+evAdApIbtB0J2AoquTV1z8GV8bqlB35WbaAOIO4Zcw5wrtiDPnMopIM+oKRH6VozNu8uYAf+Sj/dQ9EeH+jJPAD6DGo188hBynkvXj9ASYexXWKYd4rHIz+SOHQf1I454jxrxHOW8I4S+23S4J4pia/o5dwFVCeClylp8SxEdrZXpgYgr3XVV2J2zIy7xYa2S007iSOMTTZAYznsjyNN/ZVytGz4lwS+1G+5y7Jgf/IC4IA9LeRFwqHv76RIHd7n4DuydgdlX1K3Rdxn5N9pWB6/zRIf1IIekfDmLu84D94Pc9IIBfCyoW99pdNsbLTyAbsefFN6Fu5JO3vL/goe1tt/hb1OA06YC+3HTICeW/sTn6UJSOIty3ZF3B/J3A3/5ukcVX/+9B7HyWd3esNeskn6P3IOIl4s0fyiyMKZ6t/IgyBTL/LdZbb4ZXJX93n/3IDX27gh4qEIbeahJL4C/1MlT7XEVAfpEfvDqTQLw360qBf0iAcecUZfa4GPesU/k5PBKAd/UCrQKOvczwq2+9pzbXK6q+q7Jcmf2nyOzQZx+8GJf98j+gU516p8WXG8H72yK7y24GcWzQ+Gb85kt4/NPxs5Oh2bOnfJXLsrh/BPLXdz6YlsQ8bh36car7Y1y+R/3OR381Do39e5AT+Ne7xyeMeICmw838dePEHg/W0t0Y9dVLnEb/fDmAUezZ59vZU7JF/VyQkO61tOtHQH46GvB04YuizwPHl5eWfjJs8Gt73G9TUBo07ThIiD7oHgWwUeXzkw7P51zsQtvEWx3+sySRYoGvU5UPfgJClXhiaIkiUAjhA8JPxugIk/mwa5qOs6XmK+AuM/y/BSP4QjAS7T76gEf+zaKTf4ds/elkBe7+ogH7SsUGYJz6D+DC2vM2Vkzv3w/3i02t838M5j5Nvry+BWcV5Hoev+/rrQGhfWStLDotkIZCN043jVzDq4o7t4b08h6trW5ARmGBaEbYPMxwfhMbpiwlqxATLyA3wC9Iz8Lv0/dL4vioyP7IzSMAhG3PbCIELZV6SyH2M3s5Lf96refv2Xd7ykHqepv91vTwg+Yfz7c9BdCL+dhDhj/OFV+tpIVufjTt99aF+XdanMJS4sSco9mhPPrUDRb9j4vizjSxBPAnMP9fIosg7+GJbrn1a7RenuRe7cWQEnQv1DqmXPKMYmt29yqztPK+PCIUBw7f3LCjam6Ujch9WFZ3oP4188oJ8+Go/ifsjw7K4SE37BxlP6xVzI3XtH5WIY89Rk9rAUvnlbft+v4V8nJBuJUALhMuQ0r2BPJhNBP0ylO8HzKtG4c4m/HFDiWNPDML1hM6TLsyxswS7L6m7Mv4D2gYHdb7tJ74er/4HXu57T/u+j2OEflAfHg/jKD7N1lxluXSPENg7OiatzjL+ftuC/2Bwnd2xOhDvX65JWPlj/4y833lFAo5B6n5T0fnuxEFyz0NA4eE1LJyEXCJhmPdGXvSc94ScXyoGuxRzENQ55ZJwkNM54RAsw1snsKsW3NIF7zg7ss43vBkYWeab+4S9GkMiur89K86eRJ9pB+U5VXPuI5OvKOI549GA74s5G4RnqQdNgCkU+kJRt/26wwuc28W8nGhAF/Q9CbkQFgcCHOQ6kfjqPhNfX1FOVgBSyGOmys/124IAZXFdDLi/LRcS6juCZKc+UFk7vaJfhLd3TO9BzTnpDJiLjSFPZuScm7yA7GROnqadfc+zJgCbcya7V5XdtEKCwwT77NUl93eGIi/l1JcE/Lio5FkVB097Lu9RQ55WxTyvaD9j96yWg5t+tZb97Q1vbxOOWnpNvLUdpwLujcwrk+FfPd2P6+medwv8fqf+53q6zHsC+ZOpP0dD54ki9BZU742Sjnb28NDRxh4M48W+HqBzsq04/87o+7RJ+c3w+ZVO1/Ui/2e7YY60d0fZxxqOFuK8HwenbwM4ErmbQrkxXxchP5RE3pVzH+HdGKg7sJxf+9fxwz7bv/yKFboEEFfhwyV4+DUk7d3qEUh7h3rG0cGdHmF0cKVXmNKvb96Hvn/Q23w/cN/sIL63f0j/SXyjyB0uKfoOl+/FN8PejX7cF/TRAH/WsflMgP/NYP3lQYrPQSFN3RvZXwQhe1cQgRIvyNWHID8Xk++Y+vnC5A8N6J+CJHFvzljm1yBJ3W10phDsBpKfjMhn6+o/eZz8O3G3YBp7svebQJ/J9ndM0j49CeTReUgpLNqz4eb/Owb9/C7/n9mvf+jgvTlieu7pPTMGN5v3H4djHyX2Y7C8fRrIp0ju+XEfnzK9/gP+vHvo98l0EP50e/KHTQfhj52AWRHYP0L4794t7jgOZppPYPvhevKw9tGiVhRJ/c65AOzHk6bf7w5Q+X7yMdeIeLZyj/i4pVJv7hL5Win1F6+UegNwKPXCMtcj6sxtKEK+MMjfs2wPe8cQxQdZ8jf4iDEvJMLQNMpiLIMRJHVr2Fn6hUBZnCQYnKER9rQe7YqPcIUkwaLwuCwaQWny5ApuD7ahX+DD9Lmsj9L5Xzs16Uvn/xKdf+OULvIFvwbrLVYp5AXfH750Uvs/rPLPFup+ksq/Hcn+XVr7UTuiYQR4vR4EZr1fEqIvln/+CLTzvA5U6Mk5ZsP/oTohb+IAJdkX6hoI+CMOXvGlvx8Gv+Ec0l+GwfC8n+a7kSR/wal4r0CC/BhInI+8fGFv4HBjYQn85RAEnCzCg4VFKfbl2qrgTyYSPwBOT8c9Puq4kn83mn4KJz9zWiqKsC8EjVMMAWdeMOq0GuuIHXhI3F7aRwf97EhE4G4OoCEPIPxzpoh5Bp6/Il6H/R72hrPnfdTX7p0hX55E9TfenSJeLnwEUvmw/XOPgdBRhc7r2W9V6J2KudfI+yMTYsv+DvRuB5Tzn2nXaycBPrHGv3SE6A9g9zc67KcW9vFEwH+xYJ9L8Z0i+xlDSYAeN371ud1ugGMv12bytGH+GgE0Cwq42Fn0iXJ/FgIed8Z+GAKuXeoqs49zSP9eGDDEC/s6DN4ev0JBFvbiKzHmDxqCx/7bZ8AgO54Q96+GAYqSLxj16o5Plnm5ibgfN3zCkJs5mQKSZljiD4ZNjx7h60SXn+qevRH8gdjveoL9NhBEnxzp86kbMJhHd/B1ustvFD/1Y/Hjf1j87Ds6TR+5JOgfLO05sexxm8Zlk8Zhi8avrzz6+V18T1YT/eAP+PyxdcT0/R/PuJ/kf+96IvS+IOThVJdX1hAB/Bj1VbbjXMbPt/mC/0ORv3WFEoq8YxrwL9WQ98L1T6EQuzvfDj+NkvwsCB/+FAx5h+YPXsaGIti/FiS/shL+/Xbweln6D//m1ZvLL7E/itT7hbzIryL1/kjHe7v74Uj9w2uA/3sd/h8F8HfqbgEx+csbK27nlyiWeji+/nc5/Nfa/MEO/1PmEX7u7A7qyT79Tz+74/UZ/6+t+l9b9b+26n9t1f/aqv+1Vf+/fKv+MXz48Xzw5+7VR5F3HG74derWsxjwzV3VZ6m92X89xk+PwPicg7fODf35k7f0B7B8TQT8jOb/pUdvoacxjP+i7cCfN9j1j85geNVWfNL+9/tDRu7/QOm797/fF0Szf3CzMfmOoa+HOOYK1082W9xbs1MUdq0C2FU05IcuaHrgr+ALZKYB11tcVke8ZKX7tl27hzb8/KId+9k//ow+WbZ4+budN6sXPsxqPdk0+iDFD9ov+s88y8OGT5sCvQnznDI/DW/+JukRdzv9n/3lbpZ9ZkXuhwZ/o/B+Yrr5pEu/o8PyvCv0vNNy0685Ke6+i4K3DreYAPsgWNvXuMlshwy7btwCn7Gseh3VhVc78IM32i0R/qZwqy5gBq5rcYraabVGXQkysPK4KSC3ubUsDMYgnRqAp90+IEqwxHaLAj8tBfyYJLBcKgQ/BLxtFkGHgRm+YVzAiVpHhfU4Ks+v1bqJVL+MVBmPwHUIrsEXDw80cF83wYEOr0FajR+u/VNe+BuPNLkMFUXKZVkJ9nT59ExzlR/U4YP6alD2vg56BfIXmiKB/GUAvkf6Mb/cHMsqnUWb38I6zmVeyoVlgmfxVPEdwCVN00O1jUPKVc3HN/EbOArc4jVQVnxu47Eurb0v61I2bE+7XGsyRO8ppyJDazzt8Ad++FftkPFDGyFP5RLkWq6l2zeqcVArDa8T5bpt+6ebu/aWPnijE+dAaW6LP6UGe/n4DSht34ZEbd89Db4a5GldHvhYp7e8q/G1pkSxDlsjN6dvdCwv0GQ6OLZoo7XpDZDV4U1u5b/e13HL6eCK4wfZ1Lgn1/hRTid8wetmA/h79T74UeYNwGV0wN2xbaANADegjDYspwyUU9q+rHP9yaF9oE2nOuC9fG73Rb5A/spBGhv1IF+5ueQ4I7uMACqCg3ygZEEreT4+cBNyFl/vJQ6Rco/KOg1uuHEoe3PgQBTdcHKPJDzd0yEnwD1Eo6b0Lm+xL/OAISCd6KgZwRGFJ2nFQFInTTvWuc8H6sXBd33S5sMAC/jPTPFMZqG9ATe9GdBfsdVpk5w2D5g1GVrbdEJng61oz2czyY+H/DqPUwaOu1CiH/vdVs3t2L676gFDx7EaJrXchb50IE+dqKvsEF7EnQ7M34huq9NVhqP1ht/adDEcj73SRatpR6XXeMIz4golBN/hPdNb1bBtKUFFssCOtF63sKJWscqUXGjXtZSnSbnpbVBZtFpIqruDdh0tYrrBBlSUTivADWlOme2ZF4mpsDK6MwfbzbLAT72NurMTll0larAou6SkJoPxDPoptD/JannklJahzus1J+N5MJJ3oTeJVtjSD6aCBZxCC3e3LdnUNRMfEWmFSU3eE8seTitYTE+6y7Ez83R+JqLzpS9sd0Fk+cmCmG+dltY3yQ5R4obqtCG3W/PBaIYoLKhCoaBdb1mIrKICcObclGtP3UXX3gzZNjTcRCGrM07rrQ16idJOC426mdXD21LRmcXITOh7mSz3J/2gsx35qcxM86gznRLBEI+Hm6Fgyex205sWiRpbJiieGsUhNQrmo7VZzpdWGKza2Dy1SYwkWLTJNmS43DQmQ9OlyNDg9wY4eM6RLIZm8DKahHrkkDhmszjWI3CisaG+rRIR00fBEFUQdghshhUpGbIVtvl8i1LrRFdLXeMplCJngNc1bRYppqBUCkfqUKrtlbBZRNdVO2U+WQzooTsIGZyf0rM26jd4o5UdM+VHwypANSPHZ0hWdtDOfDeWlqQwo62kU9ToUpHKRbFuJpNoxqAVTu5ckaqcTpb2JnoaUl3Ux/wJxO7E3iiIxRW1XkDR84W+nCegCfv/grVrj1YkvECJMYMgC0IaRabdcnUfWUmi0gvm/sCJQOIci/V6rMdTJOWRRTNwBUnm6KLjmQjwaR2hi2+2Y9kjemzJDrxdavXkblwLejEnmFmje/bKi6Bv5/KwRgdEv8yczmZdThxFcgplGW6IqZcP8wq0RRmAHxiZKQkqofbYwClvFqwJzdbZfi9RQPwiUN5aWk8RtDvmx0laIGsgunXHQtvLgbXcpqKwHhpklhhzIWT8be6kiMZZubgbWj2LzQsN0dHa6a0XjKLOsXEkxWmRruNSWi0CbF5YqLpAyn4cVRbpqYOmY9LLVgbark6dsVOnueDMUdlVfNzE3P58uxsZleSoHYzHRLWNQQGLZVFNc4tCBbupJwvTC+Uq7mbVWumtCF9HBHSCzruLZEP2pv10ye7WA7HX7hLzrknEuGC3+AKh+tRi2HSGUr8MuPUSmS6Z2me3+VTA+0tkmAeLGPpKQk35AdMssrKL25o2TTuUn5otqiyyZT5kNXUB9SwsOsZ6MzQXBsYsMa/wAyJ3hTSDQVtvkvRsuwxaGtXv6ZI7YvnJgCV98JzQioZsMR50aS8QxHi1sfHNIGpGbNwhQHJ7qOd5KZqrDoIE+dQUmMoYGbjvq3i9kVtcxJlCH0MWs6K9kicjKQy9ZXtW5lIuUIoj1LqyXjs9dTUpWvqclNOgTsbd8VqgVotVBFQGvDYwr1ztLof5Flx4hA0MLJfqlpjBQHEciOsikGi3i056U6hiZjIBGfxFnlF8UCnDmgrjdJrVFMHRPoAOt5uTjZasqO2SbW2QQsJ3k3IrblckUm4NXcZVloC6uumMzXo8pMmodGvbiLENVQyMUJvMCtMC8TwHq9nwcWCSTTMyQqRxFmtV2KGdsVJsl02+C5oy0bMcRGpc6fCEvimXU3XW6VSJPpyhHRRyXs1SpvRAjkbgd4XUo7uqogrIOAt3s60e9dNka252UYPEuuBjjUYygdydjYMeM1voCzWY6HUybARgAEwTqbpSsDCWMIBSEqpvegrTwvq0ooWDJHBNUfCU8YbtOareEDNBmEluOpF5Ek/mrc5yhi5nBl7LahcV9dF6KGR6GFrNdsQKyBIbDhjMyLdyTkTybJc2fj+AQd8wIxd8Y+PCEkB2xTkdc5DHBultNc0vKTTehjuNpOpewxbDbsG703gywIaLcWJQXozq6zVt+FnZl2aokK66XjhEEoHLFpNOhg5LlXN2825abbQVojt1MKvr3gBZD8fbcU9IrL4XRSIX6vOtJY60LhE1eqeQMWWlSm3PRLE6MFKRRply3tfZqOTmHXwyCjKTVJiBH9Xz/iSgJoSWemhLh6EyNPyCp1mqkQI/16ITto0iaEAviXDbma4roOWS3iyowQhaMbJcdsYjfdSq8hRadGOjmD1knPR4rtMfjdpUvvBGdB/uGtF8EeZYS3jaa5thFxsUeEh5G19ndbVbDcox00VYEAE0k8G6MEnBHPRRtLcdKwMBdM9zZKQAtyrvinxRRIsJxtMwxugJnK8OE2MLOGgm/EY1QoIogr68yfQJGSwVTyWiSWK3KK3uTmbiUnRIsTd19fZc2nldyiKmxjLsiq2FxIFeatEfWkOUWfgLmca0iTZmSAKzqQoaMzpmnRU0BnPG4jkrVuQgt3aK1rC1PBMZP3E2lj/HYQ47q1UibzY6P2jnkrPdcmqDlOrAz0RyqsxqWqwre75zWGSBWbGsKtKYQ6IS4zWo3JsJMUmNOFq786hfdyY1no6pqmsY1LTeqJZl70b5so9NVR3Gc1zljFEUejXeXQy0QNmuzRpveqYFx444USZAJBDi2maFdDtx1dQkHGIiRAXGWwjWMEUv7S9cQh4EiQ/f00X7oieCC1mU+NqkPWNROdNgNFGUFtpuz1YtBPGqsjNM9Zk6CEgznOxkbwyNwbAnhJI+VPvarC03PlFttsJyMs4we13kVne66WwIQ0nq3obMxy3LwhNrOGMQnzCXfU2IrKkRWQmS96NR7Vpts1tQpC7OR3VqaiVZhbSoNTXSckNctncNwUwZY8LbOiUQRmshj7aEXhB008u1zDdhODr06FqUR0kZLoChsmlERJuhkTPtUUFN167bgLhJstABjAIciHytHnujrES6W6nNDiJ1Qww0nlZ7bhDZQjtjFVtFgu6kkPHOOoJoHrm4iaDx0KZ93+XnI8JQpVGxQiI32QEI76R5NeRxoY2gcmuHrDi+zy6SIUo4cWe4GoM36mS47nHt2diXjIzzEAq0aVjKk8FMzL1kR4lajBawfWNoJushq4TuYp4mzbQM6GS0mJt6pnNMP876LjsWxwOVnYoiu1z3q+1qaph8EBfD4aCzYXfDbZKxntz2trm/BmYmFVrjouvyIdTbuqONAq7RXSTndgmzo7pka0tTMYdWKTvn0jFk0VqQB9qMzgODKZZL3MFm/ZKdtjCC7GWEOUMCzl/PNMhIwuyFRNdnBUFQdHdV8J6zHiV0ZzKkhQVPuxmj8+ocYTbz7aizJhWCiXoRYMRyEVib7a6nt3ZksKaiQOyFFMZ6ilRlDNKW8fa2nAU8cPAta5cLQQcB5dK8addds5XnKuXs2GCChjt+3q0kgyA0jxZIS4sMxV1mQQKHoDgJdTlMrb1t2W8V1EA2wz4KXV00p5dyH123wpkzEoM2VeGLKJezDJO8KEhD3h6X3MbHGHE2AQG6kSPTuiMGa1CkNQ1ETPJngy3OjQaynWGTkW12urSPKV2pM6TVmE7TOieHBet3O92E3mGikYMnGSjb3BdQD0ZWEgJsJReWJoKlO24icY4xD9El7lXaSkFDybBBV9ndRAon7ixP6K2z3WpCCmXcnVn+YDYvurjQ20R4t2e0O8vK8vn1BG43FdJ2PjRhxAjDiJzMmSKm9CFPaD1BGNgDqYTz6J3dzM5JIt8Wsu+qfOUM+sw6bCLTZUyWT5RINSZsrsJuH5yPZ4bKaOzK6XLSGrDg3nDaOy1wEHIfX9RL2yJyh9pgUaK0MHPRUO0RNCt6e9eQdGGiq4XuwWl9qfKblVhEpLTfXRhVw3oHS3DQdVrZLuEsMN/GS0oQy4niDEaswbjSQoth6EFMlTz11go0afaM4KQw8JKtDvpC/G6Az53BGDMdZtIeYps5tOF4J7AjdRHkvDUrvRDxTQbTcHrV9yZV2gszkqOnY8Jb6Lu4V2PIPG0j7VUtTlnTC1ZDi3fTgR4kfN9dghoLsQZlcm4XMoPNl7tOiC9goGMmwkpAud0ShOwOCkM3rj/FQfzvdJgcWmo6HobYSChX/CzuJKO4w9MFmWB1v1cMxpXaEfWMQ1pVTChp22nPPdmWKJ2FFr3VnUqWYIcxK1aqEW39eDKrEMyKYiKQuLnR8F0Q90+HdtAeGci+ecOgyQcqvVQZbrSBg5Nen1jMRN4K/Cjulg2a7Yq+GU2xJYwtu2gXBryWPnFzaU2MelLfiitvwZWNP1vjruxE9GbYxxdpMNpgQjzg5qmijIVF6kYrq58Gxbzrr9e+SZTjuTPa6sJm1cyX3RlRFvycSjerZOF6blbI82qw2WCb2LX1Pgi1kNLGnFYIw4rG9NsTyOBxBrWrRTKKvGv2Nh1aXXJoyq3abIKYcOab0UIsZmbLaekYn3SMec8NZbmuYa+viadSe0gCx8C2tVnUqydNio88Ydd2I3uEZG193Oe5JU4PptzENIoJYICw7cGQWpuvhTWO8LjZMEkbjp6oEyhz1EiXdbC0XbMpujXtRSCEy5GJsNnA9VhcQSxjOoiSAOEFE1edLB/vJqtk06RzYm21crwuww6azylPnixmNuxzqkNrRyFAt7wEaSt2n+bL7RKfDjQSFZutzfOFYG+olbpOOmwomUg9m09TOyTkJb2VTCVtZtuptIu9ka50x55UxxMHR2baymMKF6VBOCJEuQ+56nZW2/nO2i16CkNLBUUs83ZU+ZUzpCZhd0Ug0symjQhTyiryna4xMDKUxKJqWvUqgNn+VlpUTEqEo2GhOW0K3Uou0c813pqOemY/deucXy3DheWNTRxLKSzcciULrDYvBvyIBqjWREuB3gaZxJN+N6U5bbscYGSCGGK4k9Juv73lnXUeFYDdXC/D6BboVuudIWOaokbN4TiC2RAArSkXorzKemmXXRfODnVrDNcZ11LnOx6pzFDUNvGA6RvBXDbmbNyX3agkoFC3ZWaqMBzd9ZjEmPJluaTHKzfYjJAki1bGfNsiZ64cdshdExsZ7Fiis0EcLT3f3U7r7a5Zt3NXBb0Sb0jYG268KSUHmEcn8kWnV0Xg8flqRyw0e552ONRSUrqCBmHmx72FO7FKY7ZNO85GpDYrRsPQGRXvUOh/l9WQXQdVJbI7b7VjmdreG1Rd1cXGVSzK0RjCQ0fVGI+zwjG47dwfbIQVKGpmtcg2JvUd2BFdsgsDtUyJW8JhDcXCgWHkFDPGEjhcsNK6QiQl8xpmJepG6k420CFEUmvjuFTdXysG9EdDkNdmXJozmFr2nEBnjBEuwNhUN7lkqXRWLaNrNj7D04I9J6kd3WbXgOn4amnICbZscX7U82NblNp1NYrxUb2fGIJg97kNpq3niVpNYH88G+pWldURSOnuLEvt2iuHDCp3TYlI3ZoRjjkhq+0wgmMJWxFNFw7ZJKMJBUsqMZvQluMmm2r0bLttC95yVY9CrplTdl3toJ3V+k2zQTJu3HcmtDVNmyE76Zk1TQjkeswXYwQPcJEZYijKb0Bv2CJmjQrMlF2YlONIMlz9YtetjGJpy0IjezlV2hqJgFBl1CH0IF9vnboc6HZeYSh0ruvCQpZWBwTBrA2b3CU2yNqOcDwgpuMJbNAyDaG4u4kgw9d30sJnyMpUDSgmtDLxuBALhXHa+a6WAMMFFto7OLOQ8GmoZiCMWuzHEgOroy2s7syzuiq1wAfBQp8FLb6/E9etXOQ74BuoIu9S4noKvmoq8i0KfK2olRCwgIE+E+a92QY+NwnRctkNQnvaqkVQxkhpVeBLanyLmKz7CPiiw6a1m6xbu5XYrat9E7jBTCU76Wbgui6cToP/f8fEJH5/AvnppJvrFXLPDm1G7zfdvGNe8ttxteXVQoLLOku8838= -------------------------------------------------------------------------------- /prometheus-advanced/system-overall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/prometheus-advanced/system-overall.png -------------------------------------------------------------------------------- /prometheus-advanced/vars/test.yml: -------------------------------------------------------------------------------- 1 | ci_cluster_id : "CL-XYZ" 2 | ingress_url_suffix: monitoring.k8s.example.com 3 | 4 | alertmanager_webhook_url: http://webhook:8080 5 | alertmanager_smtp_to: someone@someone 6 | alertmanager_smtp_from: someone@someone 7 | alertmanager_smtp_host: smtp.domain:25 8 | system_namespaces_regex: default|kube-system|pks-system|monitoring 9 | -------------------------------------------------------------------------------- /tmc-data-protection/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Enabling data protection for Kubernetes clusters on VMware Tanzu Mission Control 3 | tags: ['kubernetes', 'tanzu mission control','velero','tanzu','backup', 'tmc'] 4 | status: draft 5 | --- 6 | 7 | # Enabling data protection for Kubernetes clusters on VMware Tanzu Mission Control 8 | 9 | [VMware Tanzu Mission Control](https://tanzu.vmware.com/mission-control) is a centralized management platform for consistently operating and securing your Kubernetes infrastructure and modern applications across multiple teams and clouds. 10 | 11 | In this blog post, we will look into how *Data protection* can be enabled for Kubernetes clusters that are managed by VMware Tanzu Misson Control. In the rest of this article, we will refer to VMware Tanzu Mission Control as TMC. 12 | 13 | TMC uses [Velero](https://velero.io/) as the backup/restore tool and currently(as of 20.11.2020) AWS is the only public cloud provider that is supported for uploading the backup data. Velero is used for both kubernetes resource backup as well as persistent volume backup. For persistent volume backup, Velero is configured to use Restic and Velero is installed & configured without any user intervention when *Data protection* is enabled on a Kubernetes cluster. 14 | 15 | **NOTE**: *Contents (images/md) of this article can be found on [https://github.com/mcelep/blog/tree/master/tmc-data-protection](https://github.com/mcelep/blog/tree/master/tmc-data-protection)* 16 | 17 | ## Requirements 18 | 19 | In order to follow the tutorial, you need: 20 | 21 | - A TMC account 22 | - A Kubernetes cluster attached to your TMC account 23 | - An AWS account where you are authorized to create a cloudformation stack, IAM roles, S3 bucket 24 | - [AWS CLI](https://aws.amazon.com/cli/) 25 | - Bash shell 26 | - [jq](https://stedolan.github.io/jq/download/) : a CLI tool to parse json data 27 | 28 | 29 | ## Enable data protection on a K8S cluster 30 | 31 | You can follow the steps below to enable data protection on a K8S cluster. 32 | 33 | ### Create a data protection account on TMC 34 | 35 | As mentioned earlier, Velero will require an S3 bucket so let's have TMC provide us a [cloudformation stack](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/stacks.html). 36 | 37 | Let's create a data protection account first. Click on the *Administration* link from the menu. This should take you to a screen that looks like the image below: 38 | 39 | ![admin_account](https://github.com/mcelep/blog/blob/master/tmc-data-protection/admin_accounts.png?raw=true) 40 | 41 | Now click on *CREATE ACCOUNT CREDENTIAL* and this should bring up a drop down menu: 42 | ![account_creation_drop_down](https://github.com/mcelep/blog/blob/master/tmc-data-protection/account_creation_drop_down.png?raw=true) 43 | 44 | From this menu, select *AWS data protection credential* and it should take you to the following screen: 45 | ![create_aws_dp_provider_credential](https://github.com/mcelep/blog/blob/master/tmc-data-protection/create_aws_dp_provider_credential.png?raw=true) 46 | 47 | Give your credential a name, I've picked *mc-aws-data-protection* in this example. Before you can move to the next step, you will need to click on *GENERATE TEMPLATE*. This will initiate downloading of a cloudformation stack template file.Check your browser's designated download folder to see the newly downloaded file. 48 | 49 | Now you are in the second step of the wizard: 50 | ![create_aws_dp_provider_credential_step2](https://github.com/mcelep/blog/blob/master/tmc-data-protection/create_aws_dp_provider_credential_step2.png?raw=true) 51 | 52 | In this step, we have to create a AWS CloudFormation stack based on the template we just downloaded. We can do this either by the AWS Console or via the AWS CLI. We will do it via the AWS CLI in this step. 53 | 54 | Before we execute the AWS CLI command to execute stack creation, make sure you're logged on to your AWS account and use credentials that are authorized to create Cloudformation stacks, IAM ManagedPolicies, S3 Buckets, IAM Roles. 55 | 56 | When you run the command below, you should get some data back(an empty array or an array with stack elements): 57 | 58 | ```bash 59 | aws cloudformation describe-stacks 60 | ``` 61 | 62 | If the command above works successfully, you are ready to execute the command below: 63 | 64 | ```bash 65 | STACK_NAME=tmc-mc-data-protection aws cloudformation create-stack --template-body="$(* and ** with actual values and create a new K8S secret with the content of the file: 59 | 60 | ```bash 61 | kubectl -n velero create secret generic cloud-credentials --from-file=cloud=creds.txt 62 | ``` 63 | 64 | #### Install velero with Restic 65 | 66 | We need to opt-in for Restic installation in *values.yaml* with ```deployRestic: true``` and enable privileged mode to access Hostpath by using the following parameters: ```restic.podVolumePath``` and ```restic.privileged```. 67 | 68 | For configuring a S3 bucket, we use ```configuration.backupStorageLocation.bucket``` and ```configuration.backupStorageLocation.config.region``` parameters. Note that, we also use a parameter called ```configuration.backupStorageLocation.prefix```. This parameter comes in handy, if we are to use the same S3 bucket for multiple clusters. With the help of the prefix, we can differentiate the cluster specific content easily, so it would make sense to use a prefix that clearly identifiers a Kubernetes cluster. Before executing the command below, make sure you replace the placeholders with the right values. 69 | 70 | ```bash 71 | helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts 72 | helm install velero vmware-tanzu/velero -f values.yaml \ 73 | -n velero --version 2.13.6 \ 74 | --set configuration.backupStorageLocation.bucket= \ 75 | --set configuration.backupStorageLocation.config.region= \ 76 | --set configuration.backupStorageLocation.prefix= \ 77 | --set restic.podVolumePath=/var/lib/kubelet/pods \ 78 | --set restic.privileged=true 79 | ``` 80 | (**Note:** for TKG hostpath should be ```/var/lib/kubelet/pods```, where as for TKGI(formerly known as PKS) *restic.podVolumePath* value should read ```/var/vcap/data/kubelet/pods```) 81 | 82 | #### Create a backup && restore 83 | 84 | When using restic to backup you need to add annotations to your pods which specify the volumes to backup. See `nginx-with-pv.yaml` for an example. Here is annotation: 85 | 86 | ```txt 87 | annotations: 88 | backup.velero.io/backup-volumes: nginx-logs 89 | ``` 90 | 91 | Here are the steps to create a backup & restore from backup: 92 | 93 | ```bash 94 | # Create application resources 95 | kubectl apply -f example-app-with-pv.yaml 96 | kubectl -n example-app get pods -w # wait till pod is running 97 | # Write some data into persistent volume(PV) 98 | kubectl -n example-app exec -it "$(kubectl get pods -n example-app -o name)" -- bash -c "echo 'I persisted' > /opt/my-pvc/hi.txt" 99 | # Check if data has persisted into PV 100 | kubectl -n example-app exec -it "$(kubectl get pods -n example-app -o name)" -- bash -c "cat /opt/my-pvc/hi.txt" 101 | # Start velero backup 102 | velero backup create backup1 --include-namespaces example-app --storage-location aws --snapshot-volumes 103 | # Delete application 104 | kubectl delete namespaces example-app 105 | # Make sure PV is gone 106 | kubectl get pv -A | grep my-pvc #check no pv 107 | # Restore the latest backup 108 | velero restore create --from-backup backup1 109 | kubectl get pods -n example-app # wait till pod is running 110 | # Check if data has been restored 111 | kubectl -n example-app exec -it "$(kubectl get pods -n example-app -o name)" -- bash -c "cat /opt/my-pvc/hi.txt" 112 | ``` 113 | 114 | #### Cleanup 115 | 116 | ```bash 117 | kubectl delete ns velero 118 | kubectl delete ns example-app 119 | ``` 120 | ### 2) vSphere plugin 121 | 122 | We will cover *vSphere plugin* based velero configuration in a different blog post. 123 | 124 | ### 3) CSI Volume VolumeSnapshots 125 | 126 | We will cover *CSI VolumeSnapshots* based velero configuration in a different blog post [once it's available](https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/228). 127 | -------------------------------------------------------------------------------- /velero-1/creds.txt: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id= 3 | aws_secret_access_key= 4 | -------------------------------------------------------------------------------- /velero-1/example-app-with-pv.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 the Velero contributors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: v1 17 | kind: Namespace 18 | metadata: 19 | name: example-app 20 | labels: 21 | app: example 22 | 23 | --- 24 | kind: PersistentVolumeClaim 25 | apiVersion: v1 26 | metadata: 27 | name: my-pvc 28 | namespace: example-app 29 | labels: 30 | app: example 31 | spec: 32 | # Optional: 33 | # storageClassName: 34 | accessModes: 35 | - ReadWriteOnce 36 | resources: 37 | requests: 38 | storage: 50Mi 39 | 40 | --- 41 | apiVersion: apps/v1 42 | kind: Deployment 43 | metadata: 44 | name: example-app-deployment 45 | namespace: example-app 46 | labels: 47 | app: example 48 | spec: 49 | replicas: 1 50 | selector: 51 | matchLabels: 52 | app: example 53 | template: 54 | metadata: 55 | labels: 56 | app: example 57 | annotations: 58 | backup.velero.io/backup-volumes: my-pvc 59 | spec: 60 | volumes: 61 | - name: my-pvc 62 | persistentVolumeClaim: 63 | claimName: my-pvc 64 | containers: 65 | - image: ubuntu:bionic 66 | name: fsfreeze 67 | volumeMounts: 68 | - mountPath: "/opt/my-pvc" 69 | name: my-pvc 70 | readOnly: false 71 | command: 72 | - "/bin/bash" 73 | - "-c" 74 | - "sleep infinity" 75 | securityContext: 76 | runAsUser: 1001 77 | fsGroup: 1001 78 | 79 | -------------------------------------------------------------------------------- /velero-1/values.yaml: -------------------------------------------------------------------------------- 1 | ## 2 | ## Configuration settings that directly affect the Velero deployment YAML. 3 | ## 4 | 5 | 6 | # Init containers to add to the Velero deployment's pod spec. At least one plugin provider image is required. 7 | initContainers: 8 | - name: velero-plugin-for-aws 9 | image: velero/velero-plugin-for-aws:v1.1.0 10 | imagePullPolicy: IfNotPresent 11 | volumeMounts: 12 | - mountPath: /target 13 | name: plugins 14 | 15 | 16 | ## 17 | ## Parameters for the `default` BackupStorageLocation and VolumeSnapshotLocation, 18 | ## and additional server settings. 19 | ## 20 | configuration: 21 | # Cloud provider being used (e.g. aws, azure, gcp). 22 | provider: aws 23 | 24 | # Parameters for the `default` BackupStorageLocation. See 25 | # https://velero.io/docs/v1.0.0/api-types/backupstoragelocation/ 26 | backupStorageLocation: 27 | # Cloud provider where backups should be stored. Usually should 28 | # match `configuration.provider`. Required. 29 | name: aws 30 | provider: aws 31 | 32 | 33 | 34 | credentials: 35 | # Whether a secret should be used as the source of IAM account 36 | # credentials. Set to false if, for example, using kube2iam or 37 | # kiam to provide IAM credentials for the Velero pod. 38 | useSecret: true 39 | # Name of a pre-existing secret (if any) in the Velero namespace 40 | # that should be used to get IAM account credentials. Optional. 41 | existingSecret: cloud-credentials 42 | # Data to be stored in the Velero secret, if `useSecret` is 43 | # true and `existingSecret` is empty. This should be the contents 44 | # of your IAM credentials file. 45 | secretContents: {} 46 | # additional key/value pairs to be used as environment variables such as "DIGITALOCEAN_TOKEN: ". Values will be stored in the secret. 47 | extraEnvVars: {} 48 | 49 | # Whether to create backupstoragelocation crd, if false => do not create a default backup location 50 | backupsEnabled: true 51 | # Whether to create volumesnapshotlocation crd, if false => disable snapshot feature 52 | snapshotsEnabled: true 53 | 54 | # Whether to deploy the restic daemonset. 55 | deployRestic: true 56 | 57 | -------------------------------------------------------------------------------- /velero-1/velero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mcelep/blog/ee6965144707a28a6fa016d32c5f949d8cfc16b6/velero-1/velero.png --------------------------------------------------------------------------------