├── setup.cfg ├── .gitignore ├── .travis.yml ├── tox.ini ├── setup.py ├── LICENCE ├── tests.py ├── README.rst └── cyanite.py /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cyanite.egg-info 2 | *.pyc 3 | dist 4 | build 5 | .tox 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3.4 3 | 4 | env: 5 | - TOXENV=py26 6 | - TOXENV=py27 7 | - TOXENV=py33 8 | - TOXENV=py34 9 | - TOXENV=lint 10 | 11 | before_install: 12 | - sudo apt-get -y install libcairo2-dev 13 | 14 | install: 15 | - pip install tox 16 | 17 | script: 18 | - tox -e $TOXENV 19 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py26, 4 | py27, 5 | py33, 6 | py34, 7 | lint 8 | 9 | [testenv] 10 | setenv = 11 | PYTHONPATH={toxinidir} 12 | commands = 13 | python -Wall setup.py test 14 | deps = 15 | mock 16 | graphite-api 17 | 18 | [testenv:lint] 19 | deps = 20 | flake8 21 | commands = 22 | flake8 {toxinidir}/cyanite.py 23 | flake8 {toxinidir}/setup.py 24 | flake8 {toxinidir}/tests.py 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from setuptools import setup 3 | 4 | setup( 5 | name='cyanite', 6 | version='0.4.6', 7 | url='https://github.com/brutasse/graphite-cyanite', 8 | license='BSD', 9 | author=u'Bruno Renié', 10 | author_email='bruno@renie.fr', 11 | description=('A plugin for using graphite-web with the cassandra-based ' 12 | 'Cyanite storage backend'), 13 | long_description=open('README.rst').read(), 14 | py_modules=('cyanite',), 15 | zip_safe=False, 16 | include_package_data=True, 17 | platforms='any', 18 | classifiers=( 19 | 'Intended Audience :: Developers', 20 | 'Intended Audience :: System Administrators', 21 | 'License :: OSI Approved :: BSD License', 22 | 'Operating System :: OS Independent', 23 | 'Programming Language :: Python', 24 | 'Programming Language :: Python :: 2', 25 | 'Topic :: System :: Monitoring', 26 | ), 27 | install_requires=( 28 | 'requests', 29 | ), 30 | test_suite='tests', 31 | ) 32 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2014, Bruno Renié and contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of graphite-cyanite nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | from mock import patch 2 | from unittest import TestCase 3 | 4 | from cyanite import CyaniteFinder, chunk 5 | from graphite_api.storage import FindQuery 6 | 7 | 8 | class CyaniteTests(TestCase): 9 | def test_conf(self): 10 | config = {'cyanite': {'urls': ['http://host1:8080', 11 | 'http://host2:9090']}} 12 | CyaniteFinder(config) 13 | from cyanite import urls 14 | self.assertEqual(urls.host, 'http://host1:8080') 15 | self.assertEqual(urls.host, 'http://host2:9090') 16 | self.assertEqual(urls.host, 'http://host1:8080') 17 | 18 | @patch('requests.get') 19 | def test_metrics(self, get): 20 | get.return_value.json.return_value = [ 21 | {'path': 'foo.', 22 | 'leaf': 0}, 23 | {'path': 'foo.bar', 24 | 'leaf': 1}, 25 | ] 26 | finder = CyaniteFinder({'cyanite': {'url': 'http://host:8080'}}) 27 | query = FindQuery('foo.*', 50, 100) 28 | branch, leaf = list(finder.find_nodes(query)) 29 | self.assertEqual(leaf.path, 'foo.bar') 30 | self.assertEqual(branch.path, 'foo.') 31 | get.assert_called_once_with('http://host:8080/paths', 32 | params={'query': 'foo.*'}) 33 | 34 | get.reset_mock() 35 | get.return_value.json.return_value = { 36 | 'from': 50, 37 | 'to': 100, 38 | 'step': 1, 39 | 'series': {'foo.bar': list(range(50))}, 40 | } 41 | 42 | time_info, data = leaf.reader.fetch(50, 100) 43 | self.assertEqual(time_info, (50, 100, 1)) 44 | self.assertEqual(data, list(range(50))) 45 | 46 | get.assert_called_once_with('http://host:8080/metrics', 47 | params={'to': 100, 48 | 'path': 'foo.bar', 49 | 'from': 50}) 50 | 51 | @patch('requests.get') 52 | def test_fetch_multi(self, get): 53 | get.return_value.json.return_value = [ 54 | {'path': 'foo.baz', 55 | 'leaf': 1}, 56 | {'path': 'foo.bar', 57 | 'leaf': 1}, 58 | ] 59 | 60 | finder = CyaniteFinder({'cyanite': {'url': 'http://host:8080'}}) 61 | query = FindQuery('foo.*', 50, 100) 62 | nodes = list(finder.find_nodes(query)) 63 | 64 | get.reset_mock() 65 | get.return_value.json.return_value = { 66 | 'from': 50, 67 | 'to': 100, 68 | 'step': 1, 69 | 'series': {'foo.bar': list(range(50)), 70 | 'foo.baz': list(range(50))}, 71 | } 72 | 73 | time_info, series = finder.fetch_multi(nodes, 50, 100) 74 | self.assertEqual(set(series.keys()), set(['foo.bar', 'foo.baz'])) 75 | 76 | def test_chunk(self): 77 | mylist = range(1000, 9999) 78 | self.assertEqual(len(list(chunk(mylist, 4))), 9000) 79 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Graphite-Cyanite 2 | ================ 3 | 4 | .. image:: https://travis-ci.org/brutasse/graphite-cyanite.svg?branch=master 5 | :alt: Build Status 6 | :target: https://travis-ci.org/brutasse/graphite-cyanite 7 | 8 | A plugin for using graphite with the cassandra-based Cyanite storage 9 | backend. 10 | 11 | Requires `Graphite-API`_ **(preferred)** or Graphite-web 0.10.X. 12 | 13 | Graphite-API is available on PyPI. Read the `documentation`_ for more 14 | information. 15 | 16 | Graphite-web 0.10.X is currently unreleased. You'll need to install from 17 | source. 18 | 19 | .. _Graphite-API: https://github.com/brutasse/graphite-api 20 | .. _documentation: https://graphite-api.readthedocs.io/en/latest/ 21 | 22 | Installation 23 | ------------ 24 | 25 | :: 26 | 27 | pip install cyanite 28 | 29 | Using with graphite-api 30 | ----------------------- 31 | 32 | In your graphite-api config file:: 33 | 34 | cyanite: 35 | urls: 36 | - http://cyanite-host:port 37 | finders: 38 | - cyanite.CyaniteFinder 39 | 40 | Using with graphite-web 41 | ----------------------- 42 | 43 | In your graphite's ``local_settings.py``:: 44 | 45 | STORAGE_FINDERS = ( 46 | 'cyanite.CyaniteFinder', 47 | ) 48 | 49 | CYANITE_URLS = ( 50 | 'http://host:port', 51 | ) 52 | 53 | Where ``host:port`` is the location of the Cyanite HTTP API. If you run 54 | Cyanite on multiple hosts, specify all of them to load-balance traffic:: 55 | 56 | # Graphite-API 57 | cyanite: 58 | urls: 59 | - http://host1:port 60 | - http://host2:port 61 | 62 | # Graphite-web 63 | CYANITE_URLS = ( 64 | 'http://host1:port', 65 | 'http://host2:port', 66 | ) 67 | 68 | See `pyr/cyanite`_ for running the Cyanite carbon daemon. 69 | 70 | .. _pyr/cyanite: https://github.com/pyr/cyanite 71 | 72 | Changelog 73 | --------- 74 | 75 | * **0.4.6** (2015-10-05): Return arbitrarily large intervals in 76 | ``get_intervals()``. 77 | 78 | * **0.4.5** (2015-05-05): Handle path matches that return no data. 79 | 80 | * **0.4.4** (2014-10-22): Chunk cyanite requests if they end up being too long 81 | for cyanite. 82 | 83 | * **0.4.3** (2014-05-15): Fix a KeyError when no data is returned by cyanite 84 | for a given path. 85 | 86 | * **0.4.2** (2014-04-11): Fix graphite-web compatibility when using 87 | ``settings.CYANITE_URLS``. 88 | 89 | * **0.4.1** (2014-04-10): Fix for multiple fetches when the results are empty. 90 | 91 | * **0.4.0** (2014-04-10): Ability to fetch multiple paths at a time instead of 92 | sequentially (requires graphite-api). 93 | 94 | * **0.3.0** (2014-04-07): Change configuration syntax to allow multiple-node 95 | cyanite setups. 96 | 97 | * **0.2.1** (2014-03-07): Prevent breaking graphite rendering when no data is 98 | returned from cyanite. 99 | 100 | * **0.2.0** (2014-03-06): Graphite-API compatibility. 101 | 102 | * **0.1.0** (2013-12-08): initial version. 103 | -------------------------------------------------------------------------------- /cyanite.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | 4 | try: 5 | from graphite_api.intervals import Interval, IntervalSet 6 | from graphite_api.node import LeafNode, BranchNode 7 | except ImportError: 8 | from graphite.intervals import Interval, IntervalSet 9 | from graphite.node import LeafNode, BranchNode 10 | 11 | import requests 12 | 13 | 14 | def chunk(nodelist, length): 15 | chunklist = [] 16 | linelength = 0 17 | for node in nodelist: 18 | # the magic number 6 is because the nodes list gets padded 19 | # with '&path=' in the resulting request 20 | nodelength = len(str(node)) + 6 21 | 22 | if linelength + nodelength > length: 23 | yield chunklist 24 | chunklist = [node] 25 | linelength = nodelength 26 | else: 27 | chunklist.append(node) 28 | linelength += nodelength 29 | yield chunklist 30 | 31 | 32 | class CyaniteLeafNode(LeafNode): 33 | __fetch_multi__ = 'cyanite' 34 | 35 | 36 | class URLs(object): 37 | def __init__(self, hosts): 38 | self.iterator = itertools.cycle(hosts) 39 | 40 | @property 41 | def host(self): 42 | return next(self.iterator) 43 | 44 | @property 45 | def paths(self): 46 | return '{0}/paths'.format(self.host) 47 | 48 | @property 49 | def metrics(self): 50 | return '{0}/metrics'.format(self.host) 51 | urls = None 52 | urllength = 8000 53 | 54 | 55 | class CyaniteReader(object): 56 | __slots__ = ('path',) 57 | 58 | def __init__(self, path): 59 | self.path = path 60 | 61 | def fetch(self, start_time, end_time): 62 | data = requests.get(urls.metrics, params={'path': self.path, 63 | 'from': start_time, 64 | 'to': end_time}).json() 65 | if 'error' in data: 66 | return (start_time, end_time, end_time - start_time), [] 67 | if len(data['series']) == 0: 68 | return 69 | time_info = data['from'], data['to'], data['step'] 70 | return time_info, data['series'].get(self.path, []) 71 | 72 | def get_intervals(self): 73 | # TODO use cyanite info 74 | return IntervalSet([Interval(0, int(time.time()))]) 75 | 76 | 77 | class CyaniteFinder(object): 78 | __fetch_multi__ = 'cyanite' 79 | 80 | def __init__(self, config=None): 81 | global urls 82 | global urllength 83 | if config is not None: 84 | if 'urls' in config['cyanite']: 85 | urls = config['cyanite']['urls'] 86 | else: 87 | urls = [config['cyanite']['url'].strip('/')] 88 | if 'urllength' in config['cyanite']: 89 | urllength = config['cyanite']['urllength'] 90 | else: 91 | from django.conf import settings 92 | urls = getattr(settings, 'CYANITE_URLS') 93 | if not urls: 94 | urls = [settings.CYANITE_URL] 95 | urllength = getattr(settings, 'CYANITE_URL_LENGTH', urllength) 96 | urls = URLs(urls) 97 | 98 | def find_nodes(self, query): 99 | paths = requests.get(urls.paths, 100 | params={'query': query.pattern}).json() 101 | for path in paths: 102 | if path['leaf']: 103 | yield CyaniteLeafNode(path['path'], 104 | CyaniteReader(path['path'])) 105 | else: 106 | yield BranchNode(path['path']) 107 | 108 | def fetch_multi(self, nodes, start_time, end_time): 109 | 110 | paths = [node.path for node in nodes] 111 | data = {} 112 | for pathlist in chunk(paths, urllength): 113 | tmpdata = requests.get(urls.metrics, 114 | params={'path': pathlist, 115 | 'from': start_time, 116 | 'to': end_time}).json() 117 | if 'error' in tmpdata: 118 | return (start_time, end_time, end_time - start_time), {} 119 | 120 | if 'series' in data: 121 | data['series'].update(tmpdata['series']) 122 | else: 123 | data = tmpdata 124 | 125 | time_info = data['from'], data['to'], data['step'] 126 | return time_info, data['series'] 127 | --------------------------------------------------------------------------------