├── .gitignore ├── LICENSE ├── README.md ├── opentsdb_pandas.py └── test_opentsdb_pandas.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Tomasz Wiktor Wlodarczyk, University of Stavanger, Norway 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | opentsdb_pandas 2 | =============== 3 | 4 | This module provides methods to fetch data from [OpenTSDB](http://opentsdb.net/) HTTP interface and convert them into [Python's](http://www.python.org/) [Pandas](http://pandas.pydata.org/) [Timeseries](http://pandas.pydata.org/pandas-docs/stable/timeseries.html) object. Basic structure is based mostly on [opentsdbr](https://github.com/holstius/opentsdbr/) library. 5 | 6 | Example usage 7 | ------------- 8 | 9 | import opentsdb_pandas as opd 10 | import datetime as dt 11 | 12 | ts1 = opd.ts_get('cipsi.test1.temperature', dt.datetime(2013, 4, 3, 14, 10), dt.datetime(2013, 4, 10, 11, 30), 'node=0024C3145172746B', hostname='opentsdb.at.your.place.edu') 13 | 14 | ts1 15 | 2013-04-03 17:08:12 30.160000 16 | 2013-04-03 17:09:12 30.139999 17 | 2013-04-03 17:10:12 30.150000 18 | ... 19 | 2013-04-10 11:27:57 29.270000 20 | 2013-04-10 11:28:58 29.280001 21 | 2013-04-10 11:29:58 29.280001 22 | Length: 5514, dtype: float64 23 | 24 | help(opd) 25 | help(opd.ts_get) -------------------------------------------------------------------------------- /opentsdb_pandas.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This module provides methods to fetch data from OpenTSDB HTTP interface and convert them into Python's Pandas Timeseries object. 4 | 5 | 2014.02, Tomasz Wiktorski, University of Stavanger, Norway 6 | """ 7 | 8 | import pandas as pd 9 | import urllib2 10 | import datetime as dt 11 | 12 | 13 | def ts_get(metric, start, end, tags='', agg='avg', rate=False, downsample='', hostname='localhost', port=4242, trim=True): 14 | """ 15 | This function returns a Python's Pandas Timeseries object with data fetched from OpenTSDB basing on the provided parameters. 16 | If there are no results it returns an empty Pandas Series, in case of any other exception it throws that exception. 17 | 18 | Parameters: 19 | metric - metric name as in OpenTSDB, one metric only, e.g. 'cipsi.test1.temperature' 20 | start, end - start and end time for the query, should be of type datetime from datetime module, e.g. dt.datetime(2013, 4, 3, 14, 10), assuming: import datetime as dt 21 | tags - tags formatted according to OpenTSDB specification e.g. 'host=foo,type=user|system' 22 | agg - aggregate function to be used, default is 'avg', options are min, sum, max, avg 23 | rate - specifies if rate should be calculated instead of raw data, default False 24 | downsample - specifies downsample function and interval in OpenTSDB format, default none, e.g. '60m-avg' 25 | trim - specifies if values received from OpneTSDB should be trimed to exactly match start and end parameters, OpenTSDB by default returns additional values before the start and after the end 26 | hostname - address of OpenTSB installation, default localhost 27 | port - port of OpenTSDB installation, default 4242 28 | 29 | Example usage: 30 | import opentsdb_pandas as opd 31 | import datetime as dt 32 | ts1 = opd.ts_get('cipsi.test1.temperature', dt.datetime(2013, 4, 3, 14, 10), dt.datetime(2013, 4, 10, 11, 30), 'node=0024C3145172746B', hostname='opentsdb.at.your.place.edu') 33 | ts1 34 | """ 35 | url = "http://%s:%s/q?start=%s&end=%s&m=%s%s%s:%s{%s}&ascii" %(hostname,port,start.strftime("%Y/%m/%d-%H:%M:%S"),end.strftime("%Y/%m/%d-%H:%M:%S"), agg , ':'+downsample if downsample.strip() else '', ':rate' if rate else '', metric, tags) 36 | answer = urllib2.urlopen(url).read().strip() 37 | if answer: 38 | answer_by_line = answer.split('\n') 39 | else: 40 | return pd.Series() 41 | ti = [dt.datetime.fromtimestamp(int(x.split(' ')[1])) for x in answer_by_line] 42 | val = [float(x.split(' ')[2]) for x in answer_by_line] 43 | ts = pd.Series(val, ti) 44 | if trim: 45 | ts = ts.ix[(ts.index >= start) & (ts.index <= end)] 46 | return ts 47 | 48 | def dropcaches(hostname='localhost', port=4242): 49 | """ 50 | This function drops caches in OpenTSDB. It returns True if caches were dropped and False otherwise. 51 | 52 | Parameters: 53 | hostname - address of OpenTSB installation, default localhost 54 | port - port of OpenTSDB installation, default 4242 55 | 56 | Example usage: 57 | import opentsdb_pandas as opd 58 | opd.ts_dropcaches 59 | """ 60 | 61 | url = "http://%s:%s/dropcaches" %(hostname,port) 62 | answer = urllib2.urlopen(url).read().strip() 63 | return answer == 'Caches dropped.' 64 | -------------------------------------------------------------------------------- /test_opentsdb_pandas.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This module provides tests (pytest) for methods in opentsdb_pandas module (to fetch data from OpenTSDB HTTP interface and convert them into Pandas Timeseries object) 4 | 5 | (Most of) these tests rely on an external installation of OpenTSDB with a very specific set of data, 6 | if you test this in a different environment you will most probably have to modify the tests 7 | to reflect different OpenTSDB location, metric, etc., and of course expected results 8 | however, the general test structure should remain unchanged. 9 | 10 | 2014.02, Tomasz Wiktorski, University of Stavanger, Norway 11 | """ 12 | 13 | import opentsdb_pandas as opd 14 | import datetime as dt 15 | import pytest 16 | import urllib2 17 | 18 | def test_basic_with_trim(): #assumes all default values except hostname and tag: agg='avg', rate=False, downsample='', port=4242, trim=True 19 | ts1 = opd.ts_get('cipsi.seeds.test1.temperature', dt.datetime(2013, 04, 03), dt.datetime(2013, 04, 10), 'node=0013A2004061646F', hostname='haisen23.ux.uis.no') 20 | assert len(ts1) == 5607 21 | 22 | def test_basic_without_trim(): 23 | ts1 = opd.ts_get('cipsi.seeds.test1.temperature', dt.datetime(2013, 04, 03), dt.datetime(2013, 04, 10), 'node=0013A2004061646F', hostname='haisen23.ux.uis.no', trim=False) 24 | assert len(ts1) == 5728 25 | 26 | def test_wrong_time_span(): 27 | ts1 = opd.ts_get('cipsi.seeds.test1.temperature', dt.datetime(2015, 04, 03), dt.datetime(2015, 04, 10), 'node=0013A2004061646F', hostname='haisen23.ux.uis.no') 28 | assert len(ts1) == 0 29 | 30 | def test_wrong_metric_name(): 31 | with pytest.raises(urllib2.HTTPError): 32 | ts1 = opd.ts_get('cipsi.seeds.test1.temperaturewrong', dt.datetime(2013, 04, 03), dt.datetime(2013, 04, 10), 'node=0013A2004061646F', hostname='haisen23.ux.uis.no', trim=False) 33 | 34 | def test_wrong_tag(): 35 | with pytest.raises(urllib2.HTTPError): 36 | ts1 = opd.ts_get('cipsi.seeds.test1.temperature', dt.datetime(2013, 04, 03), dt.datetime(2013, 04, 10), 'nodes=0013A2004061646F', hostname='haisen23.ux.uis.no', trim=False) 37 | 38 | def test_wrong_host(): 39 | with pytest.raises(urllib2.URLError): 40 | ts1 = opd.ts_get('cipsi.seeds.test1.temperature', dt.datetime(2013, 04, 03), dt.datetime(2013, 04, 10), 'nodes=0013A2004061646F', hostname='haisen23.ux.ui.no', trim=False) 41 | 42 | def test_dropcaches(): 43 | t = opd.dropcaches('haisen23.ux.uis.no') 44 | assert t == True 45 | 46 | --------------------------------------------------------------------------------