├── .gitignore ├── Dockerfile ├── k8s ├── kube-api-exporter-svc.yaml └── kube-api-exporter-dep.yaml ├── Makefile ├── circle.yml ├── README.md └── main.py /.gitignore: -------------------------------------------------------------------------------- 1 | .uptodate 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM frolvlad/alpine-python3 2 | WORKDIR / 3 | # RUN apk add --update python python-dev py-pip && \ 4 | # rm -rf /var/cache/apk/* 5 | RUN pip install pykube prometheus_client 6 | COPY main.py / 7 | ENTRYPOINT ["/usr/bin/python3", "/main.py"] 8 | -------------------------------------------------------------------------------- /k8s/kube-api-exporter-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: kube-api-exporter 5 | annotations: 6 | prometheus.io/scrape: 'true' 7 | prometheus.io/path: '/' 8 | spec: 9 | ports: 10 | - port: 80 11 | name: http 12 | selector: 13 | name: kube-api-exporter 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: .uptodate 2 | 3 | IMAGE_VERSION := $(shell git rev-parse --abbrev-ref HEAD)-$(shell git rev-parse --short HEAD) 4 | 5 | .uptodate: Dockerfile main.py 6 | docker build -t tomwilkie/kube-api-exporter . 7 | docker tag tomwilkie/kube-api-exporter:latest tomwilkie/kube-api-exporter:$(IMAGE_VERSION) 8 | touch $@ 9 | 10 | clean: 11 | rm .uptodate 12 | -------------------------------------------------------------------------------- /k8s/kube-api-exporter-dep.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: kube-api-exporter 5 | spec: 6 | replicas: 1 7 | template: 8 | metadata: 9 | labels: 10 | name: kube-api-exporter 11 | spec: 12 | containers: 13 | - name: kube-api-exporter 14 | image: tomwilkie/kube-api-exporter 15 | imagePullPolicy: IfNotPresent 16 | ports: 17 | - containerPort: 80 18 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | services: 3 | - docker 4 | 5 | test: 6 | override: 7 | - make 8 | 9 | deployment: 10 | push: 11 | branch: master 12 | commands: 13 | - | 14 | docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD" && 15 | docker push tomwilkie/kube-api-exporter:latest && 16 | docker push tomwilkie/kube-api-exporter:$(git rev-parse --abbrev-ref HEAD)-$(git rev-parse --short HEAD) 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes API Exporter for Prometheus 2 | 3 | **This project is deprecated, you should use [kubernetes/kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) 4 | instead.** 5 | 6 | Kubernetes API Exporter - a Python job, delivered as a Docker image, that automatically exposes various numbers from the Kubernetes API as Prometheus metrics, such that you can alert on them. 7 | 8 | This is useful when you are continually deploying changes from a CI pipeline to a Kubernetes cluster, and you want to generate alerts when a deployments fails. 9 | 10 | For instance, the following deployment: 11 | 12 | $ kubectl get deployment helloworld -o json 13 | { 14 | "kind": "Deployment", 15 | "apiVersion": "extensions/v1beta1", 16 | "metadata": { 17 | "name": "helloworld", 18 | "namespace": "default", 19 | "generation": 2, 20 | ... 21 | }, 22 | "spec": { 23 | "replicas": 1, 24 | ... 25 | }, 26 | "status": { 27 | "observedGeneration": 2, 28 | "replicas": 1, 29 | "updatedReplicas": 1, 30 | "availableReplicas": 1 31 | } 32 | } 33 | 34 | Is translated into the following Prometheus metrics: 35 | 36 | k8s_deployment_metadata_generation{name="helloworld",namespace="default"} 2.0 37 | k8s_deployment_spec_replicas{name="helloworld",namespace="default"} 1.0 38 | k8s_deployment_spec_strategy_rollingUpdate_maxSurge{name="helloworld",namespace="default"} 1.0 39 | k8s_deployment_spec_template_spec_terminationGracePeriodSeconds{name="helloworld",namespace="default"} 30.0 40 | k8s_deployment_status_availableReplicas{name="helloworld",namespace="default"} 1.0 41 | k8s_deployment_status_observedGeneration{name="helloworld",namespace="default"} 2.0 42 | k8s_deployment_status_replicas{name="helloworld",namespace="default"} 1.0 43 | k8s_deployment_status_updatedReplicas{name="helloworld",namespace="default"} 1.0 44 | 45 | With this, you can configure the following rules to generate alerts when deployments fails: 46 | 47 | ALERT DeploymentGenerationMismatch 48 | IF k8s_deployment_status_observedGeneration{job="kube-api-exporter"} != k8s_deployment_metadata_generation{job="kube-api-exporter"} 49 | FOR 5m 50 | LABELS { severity="critical" } 51 | ANNOTATIONS { 52 | summary = "Deployment of {{$labels.exported_namespace}}/{{$labels.name}} failed", 53 | description = "Deployment of {{$labels.exported_namespace}}/{{$labels.name}} failed - observed generation != intended generation.", 54 | } 55 | 56 | ALERT DeploymentReplicasMismatch 57 | IF (k8s_deployment_spec_replicas{job="kube-api-exporter"} != k8s_deployment_status_availableReplicas{job="kube-api-exporter"}) or (k8s_deployment_spec_replicas{job="kube-api-exporter"} unless k8s_deployment_status_availableReplicas{job="kube-api-exporter"}) 58 | FOR 5m 59 | LABELS { severity="critical" } 60 | ANNOTATIONS { 61 | summary = "Deployment of {{$labels.exported_namespace}}/{{$labels.name}} failed", 62 | description = "Deployment of {{$labels.exported_namespace}}/{{$labels.name}} failed - observed replicas != intended replicas.", 63 | } 64 | 65 | # Installation 66 | 67 | To run kube-api-exporter on your own cluster, clone this repo and run: 68 | 69 | $ kubectl create -f kube-api-exporter/k8s 70 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # 3 | # Kubernetes API Exporter - expose various numbers from the Kubernetes API as 4 | # Prometheus metrics, such that you can alert on them. 5 | 6 | import numbers, optparse, time, signal, logging, sys, collections 7 | import pykube, prometheus_client, prometheus_client.core 8 | 9 | 10 | class KubernetesAPIExporter(object): 11 | 12 | KINDS = { 13 | "deployment": pykube.Deployment, 14 | "pod": pykube.Pod, 15 | "job": pykube.Job, 16 | "rc": pykube.ReplicationController, 17 | "ds": pykube.DaemonSet, 18 | } 19 | 20 | def __init__(self, api): 21 | self.api = api 22 | 23 | def collect(self): 24 | for tag, kind in self.KINDS.items(): 25 | gauge_cache = {} 26 | 27 | for thing in kind.objects(self.api).all(): 28 | labels = labels_for(thing.obj) 29 | self.record_ts_for_thing(thing.obj, labels, ["k8s", tag], gauge_cache) 30 | 31 | for gauge in gauge_cache.values(): 32 | yield gauge 33 | 34 | def pad_status_with_zero(self, value, labels, path): 35 | if path == ['k8s', 'job'] and 'status' in value and ('failed' in value['status'] or 'succeeded' in value['status']): 36 | if 'failed' not in value['status']: 37 | value['status']['failed'] = 0 38 | 39 | if 'succeeded' not in value['status']: 40 | value['status']['succeeded'] = 0 41 | 42 | def record_ts_for_thing(self, value, labels, path, gauge_cache): 43 | if isinstance(value, dict): 44 | self.pad_status_with_zero(value, labels, path) 45 | self.record_ts_for_obj(value, labels, path, gauge_cache) 46 | 47 | elif isinstance(value, list): 48 | self.record_ts_for_list(value, labels, path, gauge_cache) 49 | 50 | elif isinstance(value, numbers.Number): 51 | label_keys, label_values = zip(*labels.items()) 52 | metric_name = "_".join(path) 53 | if metric_name not in gauge_cache: 54 | gauge = prometheus_client.core.GaugeMetricFamily(metric_name, "Help text", labels=label_keys) 55 | gauge_cache[metric_name] = gauge 56 | else: 57 | gauge = gauge_cache[metric_name] 58 | gauge.add_metric(label_values, float(value)) 59 | 60 | def record_ts_for_obj(self, obj, labels, path, gauge_cache): 61 | for key, value in obj.items(): 62 | self.record_ts_for_thing(value, labels, path + [key], gauge_cache) 63 | 64 | def record_ts_for_list(self, ls, labels, path, gauge_cache): 65 | new_path, key = path[:-1], path[-1] 66 | for i, value in enumerate(ls): 67 | labels = collections.OrderedDict(labels) 68 | labels[key] = str(i) 69 | self.record_ts_for_thing(value, labels, new_path, gauge_cache) 70 | 71 | 72 | class PodLabelExporter(object): 73 | 74 | def __init__(self, api): 75 | self.api = api 76 | 77 | def collect(self): 78 | metric = prometheus_client.core.GaugeMetricFamily("k8s_pod_labels", "Timeseries with the labels for the pod, always 1.0, for joining.") 79 | for pod in pykube.Pod.objects(self.api).all(): 80 | metric.samples.append((metric.name, get_pod_labels(pod), 1.0)) 81 | yield metric 82 | 83 | 84 | class PodImageExporter(object): 85 | """Export the images that are on a pod.""" 86 | 87 | def __init__(self, api): 88 | self.api = api 89 | 90 | def collect(self): 91 | metric = prometheus_client.core.GaugeMetricFamily("k8s_pod_images", "Timeseries with spec'd images for the pod, always 1.0, for joining.") 92 | for pod in pykube.Pod.objects(self.api).all(): 93 | for image in iter_pod_images(pod): 94 | metric.samples.append((metric.name, image, 1.0)) 95 | yield metric 96 | 97 | 98 | def get_pod_labels(pod): 99 | metadata = pod.obj.get("metadata", {}) 100 | unprocessed_labels = metadata.get("labels", {}) 101 | unprocessed_labels.update({ 102 | "namespace": metadata.get("namespace", "default"), 103 | "pod_name": metadata.get("name", ""), 104 | }) 105 | return {k.replace("-", "_").replace("/", "_").replace(".", "_"): v for k, v in unprocessed_labels.items()} 106 | 107 | 108 | def iter_pod_images(pod): 109 | """Iterate through the images specified for containers in a pod.""" 110 | metadata = pod.obj.get("metadata", {}) 111 | containers = pod.obj.get("spec", {}).get("containers", []) 112 | for container in containers: 113 | yield { 114 | "namespace": metadata.get("namespace", "default"), 115 | "pod_name": metadata.get("name", ""), 116 | "container_name": container.get("name", ""), 117 | "image": container.get("image", ""), 118 | } 119 | 120 | 121 | def labels_for(obj): 122 | metadata = obj.get("metadata", {}) 123 | labels = collections.OrderedDict() 124 | labels["namespace"] = metadata.get("namespace", "default") 125 | labels["name"] = metadata.get("name", "") 126 | return labels 127 | 128 | 129 | def sigterm_handler(_signo, _stack_frame): 130 | sys.exit(0) 131 | 132 | 133 | def main(): 134 | logging.basicConfig(level=logging.INFO) 135 | 136 | parser = optparse.OptionParser("""usage: %prog [options]""") 137 | parser.add_option("--port", 138 | dest="port", default=80, type="int", 139 | help="Port to serve HTTP interface") 140 | (options, args) = parser.parse_args() 141 | 142 | logging.info("Listening on %d", options.port) 143 | 144 | api = pykube.HTTPClient(pykube.KubeConfig.from_service_account()) 145 | api.config.contexts[api.config.current_context]["namespace"] = None # Hack to fetch objects from all namespaces 146 | prometheus_client.REGISTRY.register(KubernetesAPIExporter(api)) 147 | prometheus_client.REGISTRY.register(PodLabelExporter(api)) 148 | prometheus_client.REGISTRY.register(PodImageExporter(api)) 149 | prometheus_client.start_http_server(options.port) 150 | 151 | signal.signal(signal.SIGTERM, sigterm_handler) 152 | while True: 153 | time.sleep(1) 154 | 155 | 156 | if __name__ == "__main__": 157 | main() 158 | --------------------------------------------------------------------------------