├── vadvisor ├── __init__.py ├── app │ ├── __init__.py │ ├── tree.py │ ├── statsd.py │ ├── hawkular.py │ ├── prometheus.py │ └── rest.py ├── store │ ├── __init__.py │ ├── collector.py │ └── event.py ├── virt │ ├── __init__.py │ ├── conn.py │ ├── parser.py │ ├── event.py │ ├── collector.py │ └── loop.py └── vadvisor.py ├── setup.cfg ├── docker ├── entrypoint.sh ├── cadvisor_config.json └── run-devel.sh ├── requirements-dev.txt ├── .gitignore ├── tox.ini ├── requirements.txt ├── tests ├── virt │ ├── test_parser.py │ ├── test_conn.py │ ├── vm.json │ └── vm.xml ├── app │ ├── test_metrics.py │ └── test_events.py └── store │ └── test_event.py ├── .travis.yml ├── setup.py ├── Dockerfile ├── README.md └── LICENSE /vadvisor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vadvisor/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vadvisor/store/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vadvisor/virt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | /usr/bin/vAdvisor $@ 5 | -------------------------------------------------------------------------------- /docker/cadvisor_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoint" : "http://localhost:8181/metrics" 3 | } 4 | 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -rrequirements.txt 2 | WebTest 3 | pytest 4 | pytest-cov 5 | freezegun 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .ropeproject 3 | *.swp 4 | *.egg-info 5 | .cache 6 | .coverage 7 | .tox 8 | .egg* 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py34 3 | [testenv] 4 | deps=-rrequirements-dev.txt 5 | commands=python setup.py test --addopts '--cov vadvisor' 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | libvirt-python 3 | prometheus_client 4 | six 5 | gevent >= 1.1.0 6 | wsgigzip >= 0.1.4 7 | python-dateutil 8 | argparse 9 | geventhttpclient >= 1.3.0 10 | -------------------------------------------------------------------------------- /tests/virt/test_parser.py: -------------------------------------------------------------------------------- 1 | from vadvisor.virt.parser import parse_domain_xml 2 | import json 3 | 4 | 5 | def test_xml(): 6 | result = parse_domain_xml(open("tests/virt/vm.xml").read()) 7 | assert result == json.loads(open("tests/virt/vm.json").read()) 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: python 4 | before_install: 5 | - sudo apt-get install -y $DEPS 6 | - sudo pip install coveralls 7 | install: "pip install -r requirements-dev.txt" 8 | script: python setup.py test --addopts '--cov vadvisor' 9 | matrix: 10 | include: 11 | - python: "2.7" 12 | env: DEPS="pkg-config python-dev libvirt-dev" 13 | - python: "3.4" 14 | env: DEPS="pkg-config libvirt-dev python3-dev" 15 | after_success: 16 | - coveralls 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup(name='vAdvisor', 5 | version='0.0.1', 6 | description='', 7 | long_description='', 8 | author='Roman Mohr', 9 | author_email='roman@redhat.at', 10 | url='', 11 | license="GPLv3", 12 | py_modules=[], 13 | packages=['vadvisor', 'vadvisor/app', 'vadvisor/virt', 'vadvisor/store'], 14 | setup_requires=['pytest-runner'], 15 | tests_require=['pytest', 'pytest-cov'], 16 | entry_points=""" 17 | [console_scripts] 18 | vAdvisor=vadvisor.vadvisor:run 19 | """) 20 | -------------------------------------------------------------------------------- /docker/run-devel.sh: -------------------------------------------------------------------------------- 1 | docker stop vadvisor 2 | docker stop cadvisor 3 | docker rm vadvisor 4 | docker rm cadvisor 5 | docker network rm testnet 6 | docker network create testnet 7 | 8 | sudo docker run \ 9 | --volume=/:/rootfs:ro \ 10 | --volume=/var/run:/var/run:rw \ 11 | --volume=/sys:/sys:ro \ 12 | --volume=/var/lib/docker/:/var/lib/docker:ro \ 13 | --volume=/cgroup:/cgroup:ro \ 14 | --publish=8080:8080 \ 15 | --detach=true \ 16 | --name=cadvisor \ 17 | --privileged=true \ 18 | --net=testnet \ 19 | google/cadvisor:latest 20 | 21 | sleep 10 22 | 23 | sudo docker run \ 24 | --volume=/var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro:Z \ 25 | --name vadvisor \ 26 | --publish 8181:8181 \ 27 | --detach=true \ 28 | --privileged=true \ 29 | --net=testnet \ 30 | --label io.cadvisor.metric.prometheus-vadvisor="/var/vadvisor/cadvisor_config.json" \ 31 | virtkube/vadvisor:latest 32 | -------------------------------------------------------------------------------- /vadvisor/virt/conn.py: -------------------------------------------------------------------------------- 1 | import libvirt 2 | import inspect 3 | 4 | 5 | class LibvirtConnection: 6 | 7 | ignore_codes = set([ 8 | libvirt.VIR_ERR_NO_DOMAIN, # Domain not found 9 | libvirt.VIR_ERR_NO_NETWORK, # Network not found 10 | ]) 11 | 12 | def __init__(self, con_str=None): 13 | self._con_str = con_str 14 | self._conn = None 15 | 16 | def __enter__(self): 17 | if not self._conn: 18 | self._conn = libvirt.openReadOnly(self._con_str) 19 | return self._conn 20 | 21 | def __exit__(self, exc_type, exc_value, traceback): 22 | if (self._conn and exc_type and inspect.isclass(exc_type) 23 | and issubclass(exc_type, libvirt.libvirtError) 24 | and exc_value.get_error_level() == libvirt.VIR_ERR_ERROR 25 | and exc_value.get_error_code() not in self.ignore_codes): 26 | try: 27 | self._conn.close() 28 | except Exception: 29 | pass 30 | 31 | self._conn = None 32 | -------------------------------------------------------------------------------- /vadvisor/store/collector.py: -------------------------------------------------------------------------------- 1 | from . import event 2 | 3 | 4 | class InMemoryStore: 5 | 6 | def __init__(self, seconds=60): 7 | self.seconds = seconds 8 | self.metrics = {} 9 | 10 | def put(self, metrics): 11 | for domain in metrics: 12 | uuid = domain['uuid'] 13 | if not self.metrics.get(uuid): 14 | self.metrics[uuid] = event.InMemoryStore(self.seconds) 15 | del domain['uuid'] 16 | del domain['name'] 17 | self.metrics[uuid].put(domain) 18 | for k in self.metrics.keys(): 19 | self.metrics[k].expire() 20 | if self.metrics[k].empty(): 21 | del self.metrics[k] 22 | 23 | def get(self, filter=None): 24 | result = {} 25 | if not filter: 26 | for k in self.metrics: 27 | result[k] = self.metrics[k].get(elements=self.seconds * 2) 28 | else: 29 | metrics = self.metrics.get(filter) 30 | if metrics: 31 | result[filter] = metrics.get(elements=self.seconds * 2) 32 | return result 33 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fedora:23 2 | 3 | MAINTAINER "Roman Mohr" 4 | 5 | ENV VERSION master 6 | 7 | EXPOSE 8181 8 | 9 | RUN dnf -y install tar libvirt-python3 && dnf clean all 10 | 11 | RUN dnf -y install python3-greenlet && dnf clean all && \ 12 | curl -LO https://github.com/gevent/gevent/releases/download/v1.1.1/gevent-1.1.1-cp34-cp34m-manylinux1_x86_64.whl && \ 13 | mv gevent-1.1.1-cp34-cp34m-manylinux1_x86_64.whl gevent-1.1.1-cp34-cp34m-linux_x86_64.whl && \ 14 | pip3 --no-cache-dir install gevent-1.1.1-cp34-cp34m-linux_x86_64.whl && \ 15 | rm -f gevent-1.1.1-cp34-cp34m-linux_x86_64.whl && \ 16 | rm -rf ~/.pip 17 | 18 | LABEL io.cadvisor.metric.prometheus-vadvisor="/var/vadvisor/cadvisor_config.json" 19 | 20 | RUN \ 21 | curl -LO https://github.com/kubevirt/vAdvisor/archive/$VERSION.tar.gz#/vAdvisor-$VERSION.tar.gz && \ 22 | tar xf vAdvisor-$VERSION.tar.gz && cd vAdvisor-$VERSION && \ 23 | sed -i '/libvirt-python/d' requirements.txt && \ 24 | pip3 --no-cache-dir install -r requirements.txt && pip3 --no-cache-dir install . && \ 25 | mkdir -p /var/vadvisor && cp docker/cadvisor_config.json /var/vadvisor/ && \ 26 | cp docker/entrypoint.sh / && \ 27 | rm -rf ~/.pip && \ 28 | cd .. && rm -rf vAdvisor-$VERSION* 29 | 30 | ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ] 31 | -------------------------------------------------------------------------------- /vadvisor/app/tree.py: -------------------------------------------------------------------------------- 1 | import six 2 | 3 | 4 | class Subtree: 5 | 6 | def __init__(self, field, elements): 7 | self._elements = {} 8 | for element in elements: 9 | self._elements[element.field] = element 10 | self.field = field 11 | 12 | def process(self, labels, domainStats, timestamp=None): 13 | for field, element in six.iteritems(self._elements): 14 | if field in domainStats: 15 | element.process(labels, domainStats[field], timestamp) 16 | 17 | def reset(self, label_keys): 18 | for _, element in six.iteritems(self._elements): 19 | element.reset(label_keys) 20 | 21 | def expose(self): 22 | for _, element in six.iteritems(self._elements): 23 | for metric in element.expose(): 24 | yield metric 25 | 26 | 27 | class Tree(Subtree): 28 | 29 | def __init__(self, label_keys, elements): 30 | Subtree.__init__(self, None, elements) 31 | self._label_keys = label_keys 32 | 33 | def process(self, labels, domainStats, timestamp=None): 34 | for field, element in six.iteritems(self._elements): 35 | if field in domainStats: 36 | element.process(labels, domainStats[field], timestamp) 37 | 38 | def reset(self): 39 | for _, element in six.iteritems(self._elements): 40 | element.reset(self._label_keys) 41 | -------------------------------------------------------------------------------- /tests/app/test_metrics.py: -------------------------------------------------------------------------------- 1 | import vadvisor 2 | import pytest 3 | import webtest 4 | from prometheus_client import REGISTRY 5 | from vadvisor.store.collector import InMemoryStore as MetricStore 6 | from vadvisor.app.prometheus import LibvirtCollector 7 | 8 | 9 | @pytest.fixture 10 | def _app(): 11 | 12 | class Collector: 13 | 14 | def __init__(self): 15 | self.metrics = [] 16 | 17 | def set_metrics(self, metrics): 18 | self.metrics = metrics 19 | 20 | def collect(self): 21 | return self.metrics 22 | 23 | app = vadvisor.app.rest.app 24 | app.collector = Collector() 25 | app.metricStore = MetricStore() 26 | REGISTRY.register(LibvirtCollector(collector=app.collector)) 27 | return app 28 | 29 | 30 | @pytest.fixture 31 | def app(_app): 32 | return webtest.TestApp(_app) 33 | 34 | 35 | def test_metrics(app): 36 | resp = app.get("/metrics") 37 | assert resp.status_int == 200 38 | assert "process_virtual_memory_bytes" in resp 39 | 40 | 41 | def test_vms(app): 42 | resp = app.get("/api/v1.0/stats").follow() 43 | assert resp.status_int == 200 44 | assert resp.json == {} 45 | 46 | 47 | def test_vm_state_report(app): 48 | app.app.collector.set_metrics([{ 49 | "uuid": "1234", 50 | "state": "Running", 51 | "diskio": [], 52 | "cpu": {"per_cpu_usage": []}, 53 | "network": {"interfaces": []} 54 | }]) 55 | resp = app.get("/metrics") 56 | assert "vm_up{uuid=\"1234\"} 1.0" in resp 57 | app.app.collector.set_metrics([]) 58 | resp = app.get("/metrics") 59 | assert "vm_up{uuid=\"1234\"} 0.0" in resp 60 | -------------------------------------------------------------------------------- /vadvisor/store/event.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from datetime import datetime, timedelta 3 | 4 | 5 | class InMemoryStore: 6 | 7 | """Naive memory based event store implementation. 8 | 9 | Don't make the history too big because we always have to go through the 10 | whole history for selecting and expiring the right entries. 11 | """ 12 | 13 | def __init__(self, seconds=60): 14 | self.seconds = seconds 15 | self.deque = collections.deque() 16 | 17 | def put(self, data): 18 | now = datetime.utcnow() 19 | self._expire(now) 20 | self.deque.append(Element(now, data)) 21 | 22 | def get(self, start_time=None, stop_time=None, elements=10): 23 | now = datetime.utcnow() 24 | if not start_time: 25 | start_time = datetime(1970, 1, 1) 26 | if not stop_time: 27 | stop_time = now 28 | self._expire(now) 29 | events = [] 30 | found = 0 31 | for event in self.deque: 32 | if event.timestamp >= start_time and event.timestamp <= stop_time: 33 | events.append(event.data) 34 | found += 1 35 | if elements and found >= elements: 36 | break 37 | elif event.timestamp > stop_time: 38 | break 39 | return events 40 | 41 | def expire(self): 42 | now = datetime.utcnow() 43 | self._expire(now) 44 | 45 | def _expire(self, now): 46 | lower_bound = now - timedelta(seconds=self.seconds) 47 | while len(self.deque) > 0 and self.deque[0].timestamp < lower_bound: 48 | self.deque.popleft() 49 | 50 | def empty(self): 51 | return len(self.deque) == 0 52 | 53 | 54 | class Element: 55 | 56 | def __init__(self, timestamp, data): 57 | self.data = data 58 | self.timestamp = timestamp 59 | -------------------------------------------------------------------------------- /vadvisor/virt/parser.py: -------------------------------------------------------------------------------- 1 | from xml.etree.ElementTree import XMLParser 2 | 3 | 4 | class GuestXmlParser: 5 | 6 | int_tags = ["currentMemory", "memory"] 7 | int_attribs = ["index", "port", "startport", "vram"] 8 | 9 | def __init__(self): 10 | self.json = {} 11 | self.stack = [self.json] 12 | self.catogory = None 13 | 14 | def start(self, tag, attrib): 15 | self.tag = tag 16 | for attr in self.int_attribs: 17 | if attrib.get(attr): 18 | attrib[attr] = int(attrib[attr]) 19 | if tag in ("devices", "clock"): 20 | self.category = tag 21 | self.stack[-1][tag] = [] 22 | self.stack.append(self.stack[-1][tag]) 23 | elif tag == "emulator": 24 | self.stack[-2][tag] = attrib 25 | self.stack.append(attrib) 26 | elif isinstance(self.stack[-1], dict): 27 | self.stack[-1][tag] = attrib 28 | self.stack.append(attrib) 29 | elif self.category == "devices": 30 | device = {"family": tag} 31 | device.update(attrib) 32 | self.stack[-1].append(device) 33 | self.stack.append(device) 34 | elif self.category == "clock": 35 | self.stack[-1].append(attrib) 36 | self.stack.append(attrib) 37 | 38 | 39 | def end(self, tag): 40 | self.stack.pop() 41 | 42 | def data(self, data): 43 | if data and data.strip(): 44 | if self.tag in self.int_tags: 45 | self.stack[-1]["value"] = int(data) 46 | else: 47 | self.stack[-1]["value"] = data 48 | 49 | def close(self): 50 | return self.json 51 | 52 | 53 | def parse_domain_xml(xml): 54 | target = GuestXmlParser() 55 | parser = XMLParser(target=target) 56 | parser.feed(xml) 57 | return parser.close() 58 | -------------------------------------------------------------------------------- /tests/store/test_event.py: -------------------------------------------------------------------------------- 1 | from vadvisor.store.event import InMemoryStore 2 | import pytest 3 | from freezegun import freeze_time 4 | from datetime import datetime, timedelta 5 | 6 | 7 | @pytest.fixture 8 | @freeze_time("2012-01-14 03:00:00") 9 | def expired_store(): 10 | store = InMemoryStore(60) 11 | # Insert old data 12 | store.put('old') 13 | store.put('old') 14 | store.put('old') 15 | return store 16 | 17 | 18 | @pytest.fixture 19 | @freeze_time("2012-01-14 03:01:30") 20 | def new_store(expired_store): 21 | # Insert newer data 22 | expired_store.put('new') 23 | expired_store.put('new') 24 | expired_store.put('new') 25 | return expired_store 26 | 27 | 28 | @pytest.fixture 29 | @freeze_time("2012-01-14 03:01:50") 30 | def newest_store(new_store): 31 | # Insert newer data 32 | new_store.put('newest') 33 | new_store.put('newest') 34 | new_store.put('newest') 35 | return new_store 36 | 37 | 38 | def test_empty_store(): 39 | store = InMemoryStore() 40 | assert store.get() == [] 41 | 42 | 43 | @freeze_time("2012-01-14 03:02:00") 44 | def test_expire_on_get(expired_store): 45 | expired_store.get() 46 | assert expired_store.get() == [] 47 | 48 | 49 | @freeze_time("2012-01-14 03:02:00") 50 | def test_get_all_new(new_store): 51 | assert new_store.get() == ['new', 'new', 'new'] 52 | 53 | 54 | @freeze_time("2012-01-14 03:02:00") 55 | def test_get_two_new(new_store): 56 | assert new_store.get(elements=2) == ['new', 'new'] 57 | 58 | 59 | @freeze_time("2012-01-14 03:02:00") 60 | def test_get_not_older_than(newest_store): 61 | events = newest_store.get( 62 | elements=2, 63 | start_time=datetime.utcnow() - timedelta(seconds=20) 64 | ) 65 | assert events == ['newest', 'newest'] 66 | 67 | 68 | @freeze_time("2012-01-14 03:02:00") 69 | def test_get_not_newer_than(newest_store): 70 | events = newest_store.get( 71 | elements=2, 72 | stop_time=datetime.utcnow() - timedelta(seconds=20) 73 | ) 74 | assert events == ['new', 'new'] 75 | -------------------------------------------------------------------------------- /tests/virt/test_conn.py: -------------------------------------------------------------------------------- 1 | import libvirt 2 | import pytest 3 | from vadvisor.virt.conn import LibvirtConnection 4 | 5 | 6 | @pytest.fixture 7 | def conn(): 8 | 9 | class Conn: 10 | 11 | def __init__(self): 12 | self.init = True 13 | 14 | def close(self): 15 | self.init = False 16 | 17 | def lookupByID(self, index): 18 | if not isinstance(index, int): 19 | raise AttributeError() 20 | if not self.init: 21 | raise err() 22 | 23 | c = LibvirtConnection() 24 | c._conn = Conn() 25 | return c 26 | 27 | 28 | def test_close_on_libvirtError(conn): 29 | try: 30 | with conn: 31 | raise err() 32 | except libvirt.libvirtError: 33 | pass 34 | 35 | assert not conn._conn 36 | 37 | 38 | def test_ignore_libvirtError_warnings(conn): 39 | try: 40 | with conn: 41 | raise err(level=libvirt.VIR_ERR_WARNING) 42 | except libvirt.libvirtError: 43 | pass 44 | 45 | assert conn._conn 46 | 47 | 48 | def test_ignore_no_domain_libvirtError(conn): 49 | try: 50 | with conn: 51 | raise err(libvirt.VIR_ERR_NO_DOMAIN) 52 | except libvirt.libvirtError: 53 | pass 54 | 55 | assert conn._conn 56 | 57 | 58 | def test_close_on_libvirtError_subclass(conn): 59 | try: 60 | with conn: 61 | class Err(libvirt.libvirtError): 62 | pass 63 | raise err(ex=Err(1)) 64 | except libvirt.libvirtError: 65 | pass 66 | 67 | assert not conn._conn 68 | 69 | 70 | def test_only_close_on_libvirtError(conn): 71 | e = None 72 | try: 73 | with conn as c: 74 | c.lookupById('not int') 75 | except AttributeError as err: 76 | e = err 77 | 78 | assert conn._conn 79 | assert e 80 | 81 | 82 | def test_handle_access_on_closed_connection(conn): 83 | try: 84 | with conn as c: 85 | c.close() 86 | c.lookupByID(1) 87 | except libvirt.libvirtError: 88 | pass 89 | 90 | assert not conn._conn 91 | 92 | 93 | def err(no=1, level=libvirt.VIR_ERR_ERROR, ex=libvirt.libvirtError(1)): 94 | ex.err = [no, 0, 0, level] 95 | return ex 96 | -------------------------------------------------------------------------------- /tests/app/test_events.py: -------------------------------------------------------------------------------- 1 | import vadvisor.app.rest 2 | import pytest 3 | import webtest 4 | from vadvisor.virt.event import create_event, LIFECYCLE_EVENTS 5 | from vadvisor.store.event import InMemoryStore 6 | from gevent import queue 7 | 8 | 9 | def event(event): 10 | return create_event( 11 | "test", 12 | "12345678-1234-5678-1234-567812345678", 13 | event, 14 | 0) 15 | 16 | 17 | @pytest.fixture 18 | def _app(): 19 | class Broker: 20 | 21 | def subscribe(self, subscriber): 22 | for idx, _ in enumerate(LIFECYCLE_EVENTS): 23 | subscriber.put(event(idx)) 24 | subscriber.put(StopIteration) 25 | 26 | def unsubscribe(self, queue): 27 | queue.put(StopIteration) 28 | 29 | app = vadvisor.app.rest.app 30 | broker = Broker() 31 | app.eventBroker = broker 32 | app.eventStore = InMemoryStore() 33 | 34 | q = queue.Queue() 35 | broker.subscribe(q) 36 | for element in q: 37 | app.eventStore.put(element) 38 | 39 | return app 40 | 41 | 42 | @pytest.fixture 43 | def app(_app): 44 | return webtest.TestApp(_app) 45 | 46 | 47 | def test_get_no_events(app): 48 | resp = app.get("/api/v1.0/events", 'stream=true') 49 | assert resp.status_int == 200 50 | assert 'event_type' not in resp 51 | 52 | 53 | def test_get_all_events(app): 54 | resp = app.get('/api/v1.0/events', 'stream=true&all_events=true') 55 | assert resp.status_int == 200 56 | assert 'event_type' in resp 57 | assert len(resp.body.splitlines()) == 9 58 | 59 | 60 | def test_get_two_events(app): 61 | resp = app.get('/api/v1.0/events', 'stream=true&started_events=true&resumed_events=true') 62 | assert resp.status_int == 200 63 | assert 'Started' in resp 64 | assert 'Resumed' in resp 65 | assert len(resp.body.splitlines()) == 2 66 | 67 | 68 | def test_get_single_events(app): 69 | for event in LIFECYCLE_EVENTS: 70 | resp = app.get('/api/v1.0/events', event.lower() + '_events=true&stream=true') 71 | assert resp.status_int == 200 72 | assert event in resp 73 | assert len(resp.body.splitlines()) == 1 74 | 75 | 76 | def test_get_no_history(app): 77 | resp = app.get("/api/v1.0/events") 78 | assert resp.status_int == 200 79 | assert 'event_type' not in resp 80 | 81 | 82 | def test_get_full_history(app): 83 | resp = app.get("/api/v1.0/events", "all_events=true") 84 | assert resp.status_int == 200 85 | assert 'event_type' in resp 86 | assert len(resp.body.splitlines()) == 9 87 | -------------------------------------------------------------------------------- /vadvisor/virt/event.py: -------------------------------------------------------------------------------- 1 | import libvirt 2 | 3 | from gevent import sleep 4 | from logging import debug, error 5 | from threading import Thread 6 | from datetime import datetime 7 | 8 | from . import loop 9 | from .conn import LibvirtConnection 10 | 11 | 12 | LIFECYCLE_EVENTS = ("Defined", 13 | "Undefined", 14 | "Started", 15 | "Suspended", 16 | "Resumed", 17 | "Stopped", 18 | "Shutdown", 19 | "PMSuspended", 20 | "Crashed", 21 | ) 22 | 23 | 24 | class LibvirtEventBroker(Thread): 25 | 26 | def __init__(self, conn=LibvirtConnection()): 27 | Thread.__init__(self) 28 | self._conn = conn 29 | self._subscriptions = set() 30 | 31 | def subscribe(self, subscriber): 32 | debug("Adding subscription") 33 | self._subscriptions.add(subscriber) 34 | return subscriber 35 | 36 | def unsubscribe(self, queue): 37 | debug("Removing Subscription") 38 | queue.put(StopIteration) 39 | self._subscriptions.remove(queue) 40 | 41 | def run(self): 42 | loop.virEventLoopPureRegister() 43 | libvirt.registerErrorHandler(error_handler, self) 44 | 45 | while True: 46 | try: 47 | with self._conn as conn: 48 | conn.registerCloseCallback(connection_close_callback, self) 49 | conn.domainEventRegister(lifecycle_callback, self) 50 | loop.virEventLoopPureRun() 51 | except Exception as e: 52 | error(e) 53 | sleep(5) 54 | 55 | 56 | def connection_close_callback(conn, reason, opaque): 57 | reasonStrings = ("Error", "End-of-file", "Keepalive", "Client",) 58 | error("Connection to libvirt unexpectedly closed: %s: %s" % 59 | (conn.getURI(), reasonStrings[reason])) 60 | loop.virEventLoopPureStop() 61 | if conn is not None: 62 | conn.close() 63 | 64 | 65 | def error_handler(unused, error, listener): 66 | error(error) 67 | 68 | 69 | def lifecycle_callback(connection, domain, event, detail, listener): 70 | debug("event received") 71 | e = create_event(domain.name(), domain.UUIDString(), event, detail) 72 | for subscriber in listener._subscriptions: 73 | subscriber.put(e) 74 | 75 | 76 | def create_event(name, uuid, event, reason): 77 | return { 78 | 'domain_name': name, 79 | 'domain_id': uuid, 80 | 'timestamp': datetime.utcnow(), 81 | 'event_type': LIFECYCLE_EVENTS[event], 82 | 'reason': domDetailToString(event, reason) 83 | } 84 | 85 | 86 | def domDetailToString(event, detail): 87 | domEventStrings = ( 88 | ("Added", "Updated"), 89 | ("Removed", ), 90 | ("Booted", "Migrated", "Restored", "Snapshot", "Wakeup"), 91 | ("Paused", "Migrated", "IOError", "Watchdog", 92 | "Restored", "Snapshot", "API error"), 93 | ("Unpaused", "Migrated", "Snapshot"), 94 | ("Shutdown", "Destroyed", "Crashed", 95 | "Migrated", "Saved", "Failed", "Snapshot"), 96 | ("Finished", ), 97 | ("Memory", "Disk"), 98 | ("Panicked", ), 99 | ) 100 | return domEventStrings[event][detail] 101 | -------------------------------------------------------------------------------- /vadvisor/app/statsd.py: -------------------------------------------------------------------------------- 1 | from ..virt.collector import Collector 2 | from .tree import Tree, Subtree 3 | 4 | 5 | class Metric: 6 | 7 | def __init__(self, name, field): 8 | self.field = field 9 | self.name = name 10 | self.metric = None 11 | 12 | def reset(self, label_keys): 13 | self.metric = [] 14 | 15 | def expose(self): 16 | for metric in self.metric: 17 | yield metric 18 | 19 | def process(self, labels, value, timestamp=None): 20 | name = ".".join(("vm", ".".join(labels), self.name)) 21 | self.metric.append("%s:%s|%s" % (name, str(value), self._abr)) 22 | 23 | 24 | class Gauge(Metric): 25 | 26 | _abr = "g" 27 | 28 | 29 | class Counter(Metric): 30 | 31 | _abr = "c" 32 | 33 | 34 | class Timer(Metric): 35 | 36 | _abr = "ms" 37 | 38 | 39 | class StatsdCollector: 40 | 41 | _vm = Tree(['uuid'], [ 42 | Gauge('up', 'state'), 43 | Subtree('cpu', [ 44 | Counter('cpu.total', 'cpu_time'), 45 | Counter('cpu.system.total', 'system_time'), 46 | Counter('cpu.user.total', 'user_time') 47 | ]), 48 | Subtree('memory', [ 49 | Gauge('memory.bytes', 'actual'), 50 | ]) 51 | ]) 52 | 53 | _interfaces = Tree(['uuid', 'interface'], [ 54 | Counter('network.receive.bytes.total', 'rx_bytes'), 55 | Counter('network.receive.packets.total', 'rx_packets'), 56 | Counter('network.receive.dropped.packets.total', 'rx_dropped'), 57 | Counter('network.receive.errors.total', 'rx_errors'), 58 | Counter('network.transmit.bytes.total', 'tx_bytes'), 59 | Counter('network.transmit.packets.total', 'tx_packets'), 60 | Counter('network.transmit.dropped.packets.total', 'tx_dropped'), 61 | Counter('network.transmit.errors.total', 'tx_errors'), 62 | ]) 63 | 64 | _disks = Tree(['uuid', 'device'], [ 65 | Counter('disk.write.requests.total', 'wr_reqs'), 66 | Counter('disk.write.bytes.total', 'wr_bytes'), 67 | Counter('disk.read.requests.total', 'rd_reqs'), 68 | Counter('disk.read.bytes.total', 'rd_bytes'), 69 | ]) 70 | 71 | _cpus = Tree(['uuid', 'cpu'], [ 72 | Counter('vcpu.total', 'vcpu_time'), 73 | Counter('cpu.total', 'cpu_time'), 74 | ]) 75 | 76 | def collect(self): 77 | # Get stats from libvirt 78 | stats = self.collector.collect() 79 | 80 | # Reset all metrics 81 | for tree in (self._vm, self._interfaces, self._disks, self._cpus): 82 | tree.reset() 83 | 84 | # Collect metrics 85 | for domainStats in stats: 86 | # VM status, convert state to a number 87 | domainStats['state'] = 1 if domainStats['state'] == "Running" else 0 88 | 89 | # VM stats 90 | labels = [domainStats['uuid']] 91 | self._vm.process(labels, domainStats) 92 | 93 | # Networking stats 94 | for interface in domainStats['network']['interfaces']: 95 | labels = [domainStats['uuid'], interface['name']] 96 | self._interfaces.process(labels, interface) 97 | # Disk stats 98 | for disk in domainStats['diskio']: 99 | labels = [domainStats['uuid'], disk['name']] 100 | self._disks.process(labels, disk) 101 | # CPU stats 102 | for cpu in domainStats['cpu']['per_cpu_usage']: 103 | labels = [domainStats['uuid'], str(cpu['index'])] 104 | self._cpus.process(labels, cpu) 105 | 106 | # Yield all collected metrics 107 | for tree in (self._vm, self._interfaces, self._disks, self._cpus): 108 | for metric in tree.expose(): 109 | yield metric 110 | 111 | def __init__(self, collector=Collector()): 112 | self.collector = collector 113 | -------------------------------------------------------------------------------- /vadvisor/vadvisor.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import collections 3 | from gevent import pywsgi 4 | from gevent import Greenlet, socket, sleep 5 | import logging 6 | import six 7 | from geventhttpclient import HTTPClient 8 | import json 9 | 10 | from .app.rest import make_rest_app 11 | from .virt.conn import LibvirtConnection 12 | from .app.statsd import StatsdCollector 13 | from .app.hawkular import HawkularCollector 14 | from .virt.collector import Collector 15 | 16 | 17 | def run(): 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('--port', type=int, default=8181, help='Port to serve on.') 20 | parser.add_argument('--statsd-host', type=str, required=False, help="Push VM metrics to this statsd endpoint") 21 | parser.add_argument('--statsd-port', type=int, default=8125, help="Push VM metrics to this statsd endpoint") 22 | parser.add_argument('--statsd-interval', type=int, default=15, help="Statd push interval in seconds") 23 | parser.add_argument('--hawkular-host', type=str, required=False, help="Push VM metrics to this hawkular endpoint host") 24 | parser.add_argument('--hawkular-port', type=int, default=8080, help="Push VM metrics to this hawkular endpoint port") 25 | parser.add_argument('--hawkular-interval', type=int, default=15, help="Hawkular push interval in seconds") 26 | parser.add_argument('--hawkular-tenant', type=str, default="vadvisor", help="Hawkular tenant") 27 | parser.add_argument('-v', '--verbose', action='store_true', default=False) 28 | args = parser.parse_args() 29 | 30 | logging.getLogger().setLevel(level=logging.INFO) 31 | if args.verbose: 32 | logging.getLogger('vadvisor').setLevel(level=logging.DEBUG) 33 | conn = LibvirtConnection() 34 | httpd = pywsgi.WSGIServer(('', args.port), make_rest_app(conn)) 35 | if args.statsd_host: 36 | logging.getLogger('vadvisor').info("Will push metrics to statsd endpoint %s:%s every %s seconds.", args.statsd_host, args.statsd_port, args.statsd_interval) 37 | 38 | def push_statsd_metrics(): 39 | collector = StatsdCollector(Collector(conn)) 40 | while True: 41 | try: 42 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 43 | for line in collector.collect(): 44 | socket.wait_write(sock.fileno()) 45 | sock.sendto(six.b(line), (args.statsd_host, args.statsd_port)) 46 | except Exception as e: 47 | logging.getLogger('vadvisor').error(e) 48 | sleep(args.statsd_interval) 49 | Greenlet(push_statsd_metrics).start() 50 | 51 | if args.hawkular_host: 52 | logging.getLogger('vadvisor').info("Will push metrics to hawkular endpoint %s:%s every %s seconds.", args.hawkular_host, args.hawkular_port, args.hawkular_interval) 53 | 54 | def push_hawkular_metrics(): 55 | collector = HawkularCollector(Collector(conn)) 56 | http = HTTPClient(args.hawkular_host, args.hawkular_port) 57 | while True: 58 | try: 59 | hawkular_metrics = collections.defaultdict(list) 60 | for metrics in collector.collect(): 61 | hawkular_metrics[metrics[0]].append(metrics[1]) 62 | 63 | for metrics_type, metrics in hawkular_metrics: 64 | response = http.post( 65 | '/hawkular/metrics/' + metrics_type + '/raw', 66 | json.dumps(metrics), 67 | headers={"Content-Type": "application/json", "Hawkular-Tenant": args.hawkular_tenant} 68 | ) 69 | logging.getLogger('vadvisor').debug(response) 70 | except Exception as e: 71 | logging.getLogger('vadvisor').error(e) 72 | sleep(args.hawkular_interval) 73 | Greenlet(push_hawkular_metrics).start() 74 | httpd.serve_forever() 75 | 76 | 77 | if __name__ == '__main__': 78 | run() 79 | -------------------------------------------------------------------------------- /tests/virt/vm.json: -------------------------------------------------------------------------------- 1 | {"domain": {"on_poweroff": {"value": "destroy"}, "resource": {"partition": {"value": "/machine"}}, "uuid": {"value": "eb9750b3-82e1-4afd-8a00-5efd63a36832"}, "currentMemory": {"value": 1048576, "unit": "KiB"}, "vcpu": {"placement": "static", "value": "1"}, "os": {"boot": {"dev": "hd"}, "type": {"machine": "pc-i440fx-2.4", "arch": "x86_64", "value": "hvm"}}, "clock": [{"name": "rtc", "tickpolicy": "catchup"}, {"name": "pit", "tickpolicy": "delay"}, {"name": "hpet", "present": "no"}], "devices": [{"target": {"bus": "virtio", "dev": "vda"}, "family": "disk", "alias": {"name": "virtio-disk0"}, "driver": {"type": "qcow2", "name": "qemu"}, "source": {"file": "/var/lib/libvirt/images/debian8.qcow2"}, "address": {"slot": "0x04", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x0"}, "device": "disk", "type": "file", "backingStore": {}}, {"target": {"bus": "ide", "dev": "hda"}, "family": "disk", "driver": {"type": "raw", "name": "qemu"}, "alias": {"name": "ide0-0-0"}, "readonly": {}, "address": {"bus": "0", "controller": "0", "type": "drive", "target": "0", "unit": "0"}, "device": "cdrom", "type": "file", "backingStore": {}}, {"index": 0, "family": "controller", "alias": {"name": "usb"}, "address": {"slot": "0x05", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x7"}, "model": "ich9-ehci1", "type": "usb"}, {"index": 0, "family": "controller", "alias": {"name": "usb"}, "master": {"startport": 0}, "address": {"slot": "0x05", "function": "0x0", "domain": "0x0000", "bus": "0x00", "multifunction": "on", "type": "pci"}, "model": "ich9-uhci1", "type": "usb"}, {"index": 0, "family": "controller", "alias": {"name": "usb"}, "master": {"startport": 2}, "address": {"slot": "0x05", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x1"}, "model": "ich9-uhci2", "type": "usb"}, {"index": 0, "family": "controller", "alias": {"name": "usb"}, "master": {"startport": 4}, "address": {"slot": "0x05", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x2"}, "model": "ich9-uhci3", "type": "usb"}, {"index": 0, "model": "pci-root", "type": "pci", "family": "controller", "alias": {"name": "pci.0"}}, {"index": 0, "address": {"slot": "0x01", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x1"}, "type": "ide", "family": "controller", "alias": {"name": "ide"}}, {"target": {"dev": "vnet0"}, "family": "interface", "alias": {"name": "net0"}, "source": {"bridge": "virbr0", "network": "default"}, "mac": {"address": "52:54:00:08:b6:2c"}, "address": {"slot": "0x03", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x0"}, "model": {"type": "virtio"}, "type": "network"}, {"source": {"path": "/dev/pts/6"}, "type": "pty", "target": {"port": 0}, "family": "serial", "alias": {"name": "serial0"}}, {"tty": "/dev/pts/6", "target": {"type": "serial", "port": 0}, "family": "console", "alias": {"name": "serial0"}, "source": {"path": "/dev/pts/6"}, "type": "pty"}, {"bus": "usb", "type": "tablet", "family": "input", "alias": {"name": "input0"}}, {"bus": "ps2", "type": "mouse", "family": "input"}, {"bus": "ps2", "type": "keyboard", "family": "input"}, {"autoport": "yes", "type": "vnc", "port": 5900, "family": "graphics", "listen": {"type": "address", "address": "127.0.0.1"}}, {"alias": {"name": "video0"}, "model": {"heads": "1", "vram": 16384, "type": "cirrus"}, "family": "video", "address": {"slot": "0x02", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x0"}}, {"alias": {"name": "balloon0"}, "model": "virtio", "family": "memballoon", "address": {"slot": "0x06", "bus": "0x00", "domain": "0x0000", "type": "pci", "function": "0x0"}}], "cpu": {"model": {"fallback": "allow", "value": "Broadwell"}, "mode": "custom", "match": "exact"}, "on_reboot": {"value": "restart"}, "seclabel": {"relabel": "yes", "model": "selinux", "type": "dynamic", "imagelabel": {"value": "system_u:object_r:svirt_image_t:s0:c291,c406"}, "label": {"value": "system_u:system_r:svirt_t:s0:c291,c406"}}, "pm": {"suspend-to-mem": {"enabled": "no"}, "suspend-to-disk": {"enabled": "no"}}, "features": {"acpi": {}, "apic": {}}, "emulator": {"value": "/usr/bin/qemu-kvm"}, "memory": {"value": 1048576, "unit": "KiB"}, "on_crash": {"value": "restart"}, "type": "kvm", "id": "1", "name": {"value": "debian8"}}} 2 | -------------------------------------------------------------------------------- /vadvisor/app/hawkular.py: -------------------------------------------------------------------------------- 1 | from ..virt.collector import Collector 2 | from .tree import Tree, Subtree 3 | 4 | import time 5 | 6 | 7 | class Metric: 8 | 9 | def __init__(self, name, field, unit=None): 10 | self.field = field 11 | self.name = name 12 | self.metric = None 13 | self.unit = unit 14 | 15 | def reset(self, label_keys): 16 | self.metric = [] 17 | self.label_keys = label_keys 18 | 19 | def expose(self): 20 | if not self.metric: 21 | return 22 | yield ( 23 | self.family, 24 | { 25 | "id": self.name, 26 | "data": self.metric 27 | } 28 | ) 29 | 30 | def process(self, labels, value, timestamp): 31 | data = { 32 | "timestamp": timestamp, 33 | "value": value, 34 | "tags": {k: v for k, v in zip(self.label_keys, labels)} 35 | } 36 | if self.unit: 37 | data["tags"]["unit"] = self.unit 38 | self.metric.append(data) 39 | 40 | 41 | class Gauge(Metric): 42 | 43 | family = "gauges" 44 | 45 | 46 | class Availability(Metric): 47 | 48 | family = "availability" 49 | 50 | 51 | class Counter(Metric): 52 | 53 | family = "counters" 54 | 55 | 56 | class HawkularCollector: 57 | 58 | _vm = Tree(['uuid'], [ 59 | Availability('vm_up', 'state'), 60 | Subtree('cpu', [ 61 | Counter('vm_cpu_total', 'cpu_time', 'ms'), 62 | Counter('vm_cpu_system_total', 'system_time', 'ms'), 63 | Counter('vm_cpu_user_total', 'user_time', 'ms') 64 | ]), 65 | Subtree('memory', [ 66 | Gauge('vm_memory', 'actual', 'bytes'), 67 | ]) 68 | ]) 69 | 70 | _interfaces = Tree(['uuid', 'interface'], [ 71 | Counter('vm_network_receive_total', 'rx_bytes', 'bytes'), 72 | Counter('vm_network_receive_total', 'rx_packets', 'packets'), 73 | Counter('vm_network_receive_dropped_total', 'rx_dropped', 'packets'), 74 | Counter('vm_network_receive_errors_total', 'rx_errors'), 75 | Counter('vm_network_transmit_total', 'tx_bytes', 'bytes'), 76 | Counter('vm_network_transmit_total', 'tx_packets', 'packets'), 77 | Counter('vm_network_transmit_dropped_total', 'tx_dropped', 'packets'), 78 | Counter('vm_network_transmit_errors_total', 'tx_errors'), 79 | ]) 80 | 81 | _disks = Tree(['uuid', 'device'], [ 82 | Counter('vm_disk_write_requests_total', 'wr_reqs'), 83 | Counter('vm_disk_write_total', 'wr_bytes', 'bytes'), 84 | Counter('vm_disk_read_requests_total', 'rd_reqs'), 85 | Counter('vm_disk_read_total', 'rd_bytes', 'bytes'), 86 | ]) 87 | 88 | _cpus = Tree(['uuid', 'cpu'], [ 89 | Counter('vm_vcpu_total', 'vcpu_time', 'ms'), 90 | Counter('vm_cpu_total', 'cpu_time', 'ms'), 91 | ]) 92 | 93 | def collect(self): 94 | # Get stats from libvirt 95 | stats = self.collector.collect() 96 | timestamp = int(time.time()) 97 | 98 | # Reset all metrics 99 | for tree in (self._vm, self._interfaces, self._disks, self._cpus): 100 | tree.reset() 101 | 102 | # Collect metrics 103 | for domainStats in stats: 104 | # VM status, convert state to a number 105 | domainStats['state'] = "up" if domainStats['state'] == "Running" else "down" 106 | 107 | # VM stats 108 | labels = [domainStats['uuid']] 109 | self._vm.process(labels, domainStats, timestamp) 110 | 111 | # Networking stats 112 | for interface in domainStats['network']['interfaces']: 113 | labels = [domainStats['uuid'], interface['name']] 114 | self._interfaces.process(labels, interface, timestamp) 115 | # Disk stats 116 | for disk in domainStats['diskio']: 117 | labels = [domainStats['uuid'], disk['name']] 118 | self._disks.process(labels, disk, timestamp) 119 | # CPU stats 120 | for cpu in domainStats['cpu']['per_cpu_usage']: 121 | labels = [domainStats['uuid'], str(cpu['index'])] 122 | self._cpus.process(labels, cpu, timestamp) 123 | 124 | # Yield all collected metrics 125 | for tree in (self._vm, self._interfaces, self._disks, self._cpus): 126 | for metric in tree.expose(): 127 | yield metric 128 | 129 | def __init__(self, collector=Collector()): 130 | self.collector = collector 131 | -------------------------------------------------------------------------------- /tests/virt/vm.xml: -------------------------------------------------------------------------------- 1 | 2 | debian8 3 | eb9750b3-82e1-4afd-8a00-5efd63a36832 4 | 1048576 5 | 1048576 6 | 1 7 | 8 | /machine 9 | 10 | 11 | hvm 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | Broadwell 20 | 21 | 22 | 23 | 24 | 25 | 26 | destroy 27 | restart 28 | restart 29 | 30 | 31 | 32 | 33 | 34 | /usr/bin/qemu-kvm 35 | 36 | 37 | 38 | 39 | 40 | 41 |
42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
50 | 51 | 52 | 53 |
54 | 55 | 56 | 57 | 58 |
59 | 60 | 61 | 62 | 63 |
64 | 65 | 66 | 67 | 68 |
69 | 70 | 71 | 72 | 73 | 74 | 75 |
76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 |
84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 |