├── .gitignore ├── Dockerfile ├── INSTALL-NOTES ├── LICENSE ├── README.md ├── conf ├── graphite_shim.conf.example └── local_settings_example.py ├── es-graphite-shim ├── VERSION ├── __init__.py ├── es-graphite-shim │ ├── __init__.py │ ├── admin.py │ ├── models.py │ ├── query_formatter.py │ ├── settings.py │ ├── tests.py │ ├── urls.py │ ├── views.py │ └── wsgi.py ├── lib │ ├── __init__.py │ ├── get_es_metadata.py │ └── mappings │ │ └── __init__.py ├── manage.py ├── storage │ ├── .gitkeep │ └── __init__.py └── templates │ ├── 404.html │ ├── 500.html │ ├── __init__.py │ ├── base.html │ └── index.html └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *~ 3 | *.egg 4 | *.egg-info 5 | dist 6 | build 7 | eggs 8 | venv/ 9 | *.sqlite3 10 | dumps/ 11 | es_mappings.json 12 | local_settings.py 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # DEPLOYMENT INSTRUCTIONS 2 | 3 | # To build the image, refer: 4 | # docker build -t egs_shim . 5 | 6 | # To run using the container, refer the following command: 7 | # docker run --privileged -it -d \ 8 | # -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ 9 | # -p 9000:9000 -p 9001:80 egs_shim 10 | 11 | # this port should be same as one configured for the shim 12 | ########################################################### 13 | 14 | FROM centos 15 | MAINTAINER arcolife 16 | 17 | # Install useful packages 18 | RUN yum install -y procps-ng tar vim git wget gcc 19 | 20 | RUN wget https://www.softwarecollections.org/en/scls/rhscl/python33/epel-7-x86_64/download/rhscl-python33-epel-7-x86_64.noarch.rpm 21 | RUN yum clean all; yum install -y rhscl-python33-epel-7-x86_64.noarch.rpm; yum -y install python33 22 | 23 | RUN yum install -y mod_wsgi httpd 24 | 25 | # Clone the shim 26 | # RUN git clone https://github.com/distributed-system-analysis/es-graphite-shim.git /opt/es-graphite-shim 27 | 28 | RUN mkdir -p /opt/es-graphite-shim/es-graphite-shim/ 29 | ADD es-graphite-shim/ /opt/es-graphite-shim/es-graphite-shim/ 30 | RUN mkdir /opt/es-graphite-shim/logs/; touch /opt/es-graphite-shim/logs/error.log; touch /opt/es-graphite-shim/logs/access.log 31 | COPY requirements.txt /opt/es-graphite-shim/ 32 | COPY conf/local_settings.py /opt/es-graphite-shim/es-graphite-shim/ 33 | 34 | WORKDIR /opt/es-graphite-shim 35 | 36 | # RUN easy_install pip 37 | # RUN virtualenv venv 38 | # RUN source venv/bin/activate; pip install -r /opt/es-graphite-shim/requirements.txt 39 | RUN scl enable python33 "easy_install-3.3 pip" 40 | 41 | # RUN scl enable python33 "pip3.3 install virtualenv" 42 | # RUN scl enable python33 "virtualenv venv; source venv/bin/activate; pip3.3 install -r /opt/es-graphite-shim/requirements.txt; bash" 43 | RUN scl enable python33 "pip3.3 install -r /opt/es-graphite-shim/requirements.txt" 44 | 45 | # add httpd configs 46 | COPY conf/graphite_shim.conf.example /etc/httpd/conf.d/graphite_shim.conf 47 | RUN sed -i s#Listen\ 80#Listen\ 80\\nListen\ 9000#g /etc/httpd/conf/httpd.conf 48 | 49 | # modify selinux policies and folder ownerships 50 | RUN chown -R apache:apache /opt/es-graphite-shim/ 51 | 52 | # Enable using systemd 53 | RUN scl enable python33 bash; systemctl enable httpd 54 | 55 | RUN echo "root:egs_shim" | chpasswd 56 | 57 | # Launch bash as the default command if none specified. 58 | CMD ["/usr/sbin/init"] 59 | -------------------------------------------------------------------------------- /INSTALL-NOTES: -------------------------------------------------------------------------------- 1 | # SPECIAL NOTE: 2 | # Stick to a PORT value and keep it constant throughout 3 | # the installation steps. (make sure it doesn't collide with 4 | # already running service on the same port. 5 | # Take port based on the ones allowed under SELinux Policy 6 | # to http. Check with following command: 7 | # $ sudo semanage port -l | grep http 8 | 9 | # Recommended: Take PORT as 9000 (it's pre-allotted to http). 10 | 11 | ########################################################## 12 | # Install apache, mod_wsgi and configure it 13 | yum install httpd python-devel mod_wsgi 14 | # then add the following to httpd.conf: 15 | # Listen PORT 16 | 17 | # copy graphite_shim.conf to /etc/httpd/conf.d/ and add 18 | cp graphite_shim.conf.example /etc/httpd/conf.d/ 19 | # and make sure the is same as 20 | # defined for 'Listen' direction under httpd.conf 21 | # Also, check for following line in httpd.conf: 22 | # IncludeOptional conf.d/*.conf 23 | 24 | # under es-graphite-shim/ copy shim_local.cfg as shim.cfg 25 | # and fill necessary configurations 26 | 27 | # copy the following folders to /opt/es-graphite-shim/ 28 | mkdir /opt/es-graphite-shim/ 29 | mkdir /opt/es-graphite-shim/logs /opt/es-graphite-shim/storage 30 | cp -r es-graphite-shim/ /opt/es-graphite-shim/ 31 | 32 | # Add virtual env in under /opt/es-graphite-shim/ 33 | cd /opt/es-graphite-shim/ 34 | virtualenv venv 35 | source venv/bin/activate 36 | 37 | # Install requirements 38 | pip install -r requirements.txt 39 | # (Optional) Deactivate venv 40 | deactivate 41 | 42 | #NOTE: example local config is given under: 43 | # /opt/es-graphite-shim/conf/local_settings_example.py 44 | # defaut: http://localhost:9200 45 | cp /opt/es-graphite-shim/conf/local_settings_example.py \ 46 | /opt/es-graphite-shim/es-graphite-shim/local_settings.py 47 | # After this, add your elasticsearch instance and other configurations 48 | # in /opt/es-graphite-shim/es-graphite-shim/local_settings.py 49 | # - DB field is optional and is not used currently. 50 | # provide the hostname as the host where the shim is to be installed. 51 | # - the FRESH field is for an internal functionality. Let it remain as 52 | # default value in example config. 53 | # - Don't forget to modify SECRET_KEY field as well. 54 | # - INDEX_PREFIX and DOC_TYPE are used to indicate the index name 55 | # pattern. Its recommended to name your ES indices in a manner so that 56 | # selecting them while querying becomes easier, so as to avoid 57 | # querying all indices. The pattern here is: 58 | # INDEX_PREFIX.DOC_TYPE- 59 | # - Add the FIELD name as the first metric parameter in the graphite based 60 | # dot separated metric path. 61 | # Example, if we have a metric path as: log1.value1.sub1.. 62 | # then the FIELD name would consider the field type of log1. 63 | # This is used to query and provide info about first metric in grafana 64 | # in dropdown list. 65 | 66 | # create log files 67 | touch /opt/es-graphite-shim/logs/error.log 68 | touch /opt/es-graphite-shim/logs/access.log 69 | 70 | # Optional: create django db and make sure you keep 71 | # the same password under DATABASE config in: 72 | # /opt/es-graphite-shim/es-graphite-shim/local_settings.py 73 | python /opt/es-graphite-shim/es-graphite-shim/manage.py syncdb 74 | 75 | # Go to /opt/es-graphite-shim/es-graphite-shim/es-graphite-shim/settings.py 76 | # set SECRET_KEY to a big random key 77 | # set ALLOWED_HOSTS to [ 'localhost', '", 25 | timezone: 'Asia/Kolkata', 26 | }, 27 | elasticsearch: { 28 | type: 'elasticsearch', 29 | url: "", 30 | index: 'grafana-dash', 31 | grafanaDB: true, 32 | } 33 | }, 34 | 35 | .. 36 | ``` 37 | 38 | The `timezone` option under `datasources.graphite` has to be mentioned 39 | in case the timezone offsets differ. This normally has to be same as the 40 | one mentioned under `local_settings.py` (change this line: `TIME_ZONE` = '') 41 | 42 | ### USAGE 43 | 44 | To run this in development mode, just like any other 45 | django project would be executed. 46 | 47 | ``` $ python manage.py runserver ``` 48 | 49 | Or, you might deploy this on a production server. For this, 50 | refer to the sample apache based deployment help guide included 51 | within the source code, in a file called INSTALL-NOTES and sample 52 | apache config under conf/graphite_shim.conf.example 53 | 54 | Following that, on homepage, you will see some sample links. 55 | 56 | There are two categories of the shim API, as follows: 57 | 58 | 1. __Render Query Type__: Here, the metric_path is a DOT (.) separated path (similar to graphite) that specifies 59 | the path of the query metrics, as per the hierarchy. This may be: 60 | 61 | - Format: ```/render/?target={{metric_path}}&from={{epoch_from}}&until={{epoch_until}}&format=json``` 62 | 63 | ..where _metric_path_ is: ```metric1.sub_metric1.sub_sub_metric1. ``` 64 | 65 | - The ```from=``` and ```until=``` fields specify the time durations between which the query 66 | is to be performed. 67 | 68 | - ```format=json``` ensures the response is JSON formatted. 69 | 70 | - Please note, for demonstration purposes, a sample query has been provided in the homepage 71 | HTML file. The params provided to it, come from within the views.py file under 72 | ```homepage(request)```. You could modify the query path and epoch intervals accordingly. 73 | 74 | 2. __Metric Search Query Type__: Here, when * is given, all the parent nodes in the metric path hierarchy are displayed as a result, along with information like, whether its a leaf node or not. 75 | 76 | - Format: ```/metrics/find?query=*``` 77 | 78 | ### LICENSE 79 | Refer to the file 'LICENSE'. 80 | -------------------------------------------------------------------------------- /conf/graphite_shim.conf.example: -------------------------------------------------------------------------------- 1 | # This needs to be in your server's config somewhere, probably 2 | # the main httpd.conf 3 | # NameVirtualHost *:80 4 | 5 | # This line also needs to be in your server's config. 6 | # LoadModule wsgi_module modules/mod_wsgi.so 7 | 8 | 9 | LoadModule wsgi_module modules/mod_wsgi.so 10 | 11 | 12 | # XXX You need to set this up! 13 | # Read http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives#WSGISocketPrefix 14 | WSGISocketPrefix run/wsgi 15 | 16 | # NOTE: Do not forget to change the port number 17 | # here. And put the final URL of the shim in grafana's 18 | # config.js in place of graphite instance URL 19 | 20 | # ServerName localhost 21 | DocumentRoot "/opt/es-graphite-shim/" 22 | ErrorLog /opt/es-graphite-shim/logs/error.log 23 | CustomLog /opt/es-graphite-shim/logs/access.log common 24 | 25 | Header set Access-Control-Allow-Origin "*" 26 | 27 | Header set Access-Control-Allow-Methods "GET, POST, OPTIONS" 28 | Header set Access-Control-Allow-Headers "origin, authorization, accept" 29 | 30 | WSGIDaemonProcess es-graphite-shim python-path=/opt/es-graphite-shim/es-graphite-shim:/opt/rh/python33/root/lib/python3.3/site-packages/ 31 | WSGIProcessGroup es-graphite-shim 32 | WSGIApplicationGroup %{GLOBAL} 33 | WSGIImportScript /opt/es-graphite-shim/es-graphite-shim/es-graphite-shim/wsgi.py process-group=es-graphite-shim application-group=%{GLOBAL} 34 | 35 | WSGIScriptAlias / /opt/es-graphite-shim/es-graphite-shim/es-graphite-shim/wsgi.py 36 | Alias /content/ /opt/es-graphite-shim/es-graphite-shim/templates/ 37 | 38 | 39 | SetHandler None 40 | Order deny,allow 41 | Allow from all 42 | 43 | 44 | 45 | 46 | 47 | Require all granted 48 | 49 | 50 | 51 | 52 | Options All 53 | AllowOverride All 54 | Require all granted 55 | 56 | 57 | 58 | ProxySet connectiontimeout=5 timeout=90 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /conf/local_settings_example.py: -------------------------------------------------------------------------------- 1 | ES_HOST = "localhost" 2 | ES_PORT = 9200 3 | DOC_TYPE = "doc_type" 4 | INDEX_PREFIX = "index_prefix" 5 | FIELD = "field_name" 6 | 7 | FRESH = False 8 | 9 | import socket as _sock 10 | HOSTNAME = _sock.gethostname() 11 | del _sock 12 | 13 | DB_NAME = "db" 14 | DB_PASS = "1234" 15 | 16 | SECRET_KEY = "" 17 | 18 | TIME_ZONE = 'Asia/Kolkata' 19 | -------------------------------------------------------------------------------- /es-graphite-shim/VERSION: -------------------------------------------------------------------------------- 1 | 0.1.1a0 2 | -------------------------------------------------------------------------------- /es-graphite-shim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/__init__.py -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/es-graphite-shim/__init__.py -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/query_formatter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import json 4 | import re 5 | from datetime import datetime, timedelta 6 | from django.conf import settings 7 | from collections import defaultdict 8 | from time import ctime 9 | # from calendar import timegm 10 | # dt = datetime.strptime(tmp, "%Y-%m-%d %H:%M:%S") 11 | # return timegm(dt.utctimetuple()) 12 | 13 | def flatten_list(L): 14 | v = [] 15 | [v.extend(x) for x in L] 16 | return v 17 | 18 | 19 | def form_response(text='', _id='', leaf=0, context={}, expandable=1, allowChildren=1): 20 | """ 21 | Returns Grafana readable response. 22 | """ 23 | return { "leaf": leaf, 24 | "context": context, 25 | "text": text, 26 | "expandable": expandable, 27 | "id": _id, 28 | "allowChildren": allowChildren } 29 | 30 | 31 | def iterate_mappings(query=None, idx=2, metrics=None): 32 | """ 33 | Finds out metric paths iteratively 34 | """ 35 | path = query[:idx] 36 | while query[idx:]: 37 | _metric = query[idx:][0] 38 | try: 39 | metrics = metrics[_metric]['properties'] 40 | path.append(_metric) 41 | except KeyError: 42 | if _metric == '*': 43 | # is a mass query 44 | tmp = list(metrics.keys()) 45 | if 'properties' in metrics[tmp[0]]: 46 | _leaf = 0 47 | expandable = 1 48 | allowChildren = 1 49 | else: 50 | _leaf = 1 51 | expandable = 0 52 | allowChildren = 0 53 | metrics = tmp 54 | if idx == 2: 55 | metrics.remove('_metadata') 56 | metrics.remove('timestamp') 57 | return [_leaf, expandable, allowChildren, path, metrics] 58 | elif _metric in metrics: 59 | if len(query[idx:]) == 1: 60 | # inference: query doesn't have a * at the end 61 | _leaf = 1 62 | expandable = 0 63 | allowChildren = 0 64 | return [_leaf, expandable, allowChildren, path, query] 65 | else: 66 | return None 67 | # doesn't contain 'properties' 68 | else: 69 | # is invalid query 70 | return None 71 | idx += 1 72 | 73 | expandable = 1 74 | allowChildren = 1 75 | _leaf = 0 76 | return [_leaf, expandable, allowChildren, path, query] 77 | 78 | 79 | def find_metrics(query): 80 | """ 81 | API of graphite mocked: /metric/find?query= 82 | """ 83 | response = [] 84 | query = query.split('.') 85 | first_metric_list = settings._FIELDS 86 | 87 | if len(query) == 1: 88 | if query[0] == '*': 89 | # return all first_metric_list 90 | for name in first_metric_list: 91 | response.append(form_response(text=name, _id=name)) 92 | else: 93 | # if no '*' in query, validate fieldname 94 | if query[0] in first_metric_list: 95 | response.append(form_response(text=query[0], _id=query[0])) 96 | else: 97 | return json.dumps(response) 98 | 99 | elif len(query) >= 2: 100 | mappings = settings._MAPPINGS 101 | doc_types = list(mappings['mappings'].keys()) 102 | assert query[0] in first_metric_list 103 | 104 | if len(query) == 2: 105 | if query[1] == '*': 106 | # TODO: filter query for fieldname specific docs 107 | for doc_type in doc_types: 108 | response.append(form_response(text=doc_type, 109 | _id=query[0] + '.' + doc_type)) 110 | else: 111 | assert query[1] in doc_types 112 | response.append(form_response(text=query[1], 113 | _id=query[0] + '.' + query[1])) 114 | 115 | elif len(query) > 2: 116 | assert query[1] in doc_types 117 | metrics = mappings['mappings'][query[1]]['properties'] 118 | result = iterate_mappings(query=query, metrics=metrics) 119 | if result: 120 | if result[-1] == query: 121 | # query without * at the end 122 | response.append(form_response(text=query[-1], _id='.'.join(query), 123 | leaf=result[0], allowChildren=result[2], 124 | expandable=result[1])) 125 | else: 126 | # query with * at the end 127 | for item in result[-1]: 128 | response.append(form_response(text=item, _id='.'.join(result[-2]) + '.' + item, 129 | leaf=result[0], allowChildren=result[2], 130 | expandable=result[1])) 131 | else: 132 | # invalid query 133 | json.dumps(response) 134 | 135 | return json.dumps(response) 136 | 137 | 138 | def restructure_query(pos=None, tmp=[], target=None, items=[]): 139 | if tmp: 140 | for i in range(len(tmp)): 141 | k = [] 142 | for current in items: 143 | curr = tmp[i][:] 144 | curr[pos] = current 145 | k.append(curr) 146 | tmp[i] = k 147 | tmp = flatten_list(tmp) 148 | else: 149 | for current in items: 150 | _target = target[:] # copy target 151 | _target[pos] = current 152 | tmp.append(_target) 153 | 154 | return tmp 155 | 156 | 157 | def query_es(_type=None, _COUNT=10, fieldname=None, _fields=[], _from="-2d", _until="now"): 158 | _fields.append("_timestamp") 159 | 160 | if _until == 'now': 161 | timelt = datetime.now() 162 | try: 163 | timegt = datetime.utcfromtimestamp(float(_from)) 164 | except: 165 | if 'd' in _from: 166 | timegt = timelt + timedelta(days=int(_from.strip('d'))) 167 | elif 'h' in _from: 168 | timegt = timelt + timedelta(hours=int(_from.strip('h'))) 169 | elif 'min' in _from: 170 | timegt = timelt + timedelta(minutes=int(_from.strip('min'))) 171 | else: 172 | return [] 173 | else: 174 | timelt = datetime.utcfromtimestamp(float(_until)) 175 | timegt = datetime.utcfromtimestamp(float(_from)) 176 | 177 | indexes = defaultdict(int) 178 | ltday = timelt.strftime("%Y%m%d") 179 | indexes[ltday] = 1 180 | ts = timegt 181 | oneday = timedelta(days=1) 182 | while ts < timelt: 183 | indexes[ts.strftime("%Y%m%d")] += 1 184 | ts += oneday 185 | 186 | suffix_list = [suffix for suffix in list(indexes.keys())] 187 | suffix_list.sort(reverse=True) 188 | index_list = ["%s.%s-%s" % (settings.INDEX_PREFIX, settings.DOC_TYPE, suffix) for suffix in suffix_list] 189 | 190 | _body = { 191 | "size": 2**8, 192 | "query": { 193 | "filtered": { 194 | "filter": { 195 | "and": { 196 | "filters": [ 197 | { 198 | "range": { 199 | "_timestamp": { 200 | "gt": timegt.strftime("%Y-%m-%dT%H:%M:%S"), 201 | "lt": timelt.strftime("%Y-%m-%dT%H:%M:%S") 202 | }, 203 | "execution": "fielddata", 204 | "_cache": "false" 205 | } 206 | }, 207 | { 208 | "term": { 209 | settings.FIELD: fieldname, 210 | "_cache": "false" 211 | } 212 | } 213 | ], 214 | "_cache": "false" 215 | } 216 | } 217 | } 218 | } 219 | } 220 | 221 | print("[%s] - Query Metadata: %s" % \ 222 | (ctime(), _body)) 223 | _body['fields'] = _fields 224 | res = settings.ES.search(doc_type=_type, body=_body, index=index_list) 225 | _fields.remove("_timestamp") 226 | 227 | return [x['fields'] for x in res['hits']['hits']] 228 | 229 | 230 | def build_query(_doc_type, _node, fields, response, _COUNT, _FROM, _UNTIL, target): 231 | res = query_es(_type=_doc_type, _COUNT=_COUNT, 232 | fieldname=_node, _fields=list(fields), 233 | _from=_FROM, 234 | _until=_UNTIL) 235 | 236 | for _field in fields: 237 | if res: 238 | d = { 239 | "target" : '.'.join([_node,target[1],_field]), 240 | "datapoints" : [] 241 | } 242 | 243 | for current in res: 244 | d['datapoints'].append([current[_field][0], float(current['_timestamp'])/1000]) 245 | 246 | response.append(d) 247 | 248 | # else: 249 | # _init = ( int(_UNTIL) - int(_FROM) ) / 60 / 2 + int(_FROM) 250 | # for _tstamp in range(_init, _init + 60 * 20, 60): 251 | # d['datapoints'].append([None, _tstamp]) 252 | 253 | return response 254 | 255 | 256 | def render_metrics(_TARGETS, _FROM, _UNTIL, _COUNT): 257 | """ 258 | API of graphite mocked: /render/? 259 | """ 260 | response = [] 261 | first_metric_list = settings._FIELDS 262 | mappings = settings._MAPPINGS 263 | doc_types = list(mappings['mappings'].keys()) 264 | 265 | for target in _TARGETS: 266 | brace_items = re.findall(r'(\{.+?\})+', target) 267 | target = target.split('.') 268 | 269 | if brace_items: 270 | tmp = [] 271 | for item in brace_items[::-1]: 272 | try: 273 | pos = target.index(item) 274 | except: 275 | break 276 | 277 | # remove { } and then split comma delimited query 278 | items = item[:-1][1:].split(',') 279 | tmp = restructure_query(pos=pos, tmp=tmp, target=target, items=items) 280 | fields = set(['.'.join(x[2:]) for x in tmp]) 281 | 282 | else: 283 | fields = set(['.'.join(target[2:])]) 284 | 285 | fields_list = list(fields) 286 | 287 | try: 288 | # check for multiple first_metric_list and doc_types 289 | _node_check = bool(re.findall(r'(\{.+?\})+', target[0])) 290 | _doc_type_check = bool(re.findall(r'(\{.+?\})+', target[1])) 291 | 292 | # validate doc_types 293 | if _doc_type_check: 294 | _types = target[1][:-1][1:].split(',') 295 | else: 296 | _types = [target[1]] 297 | _doc_types = set(doc_types) & set(_types) 298 | if not bool(_doc_types): 299 | continue 300 | 301 | # validate fields 302 | for _doc_type in _doc_types: 303 | metrics = mappings['mappings'][_doc_type]['properties'] 304 | for _field in fields: 305 | query = _field.split('.') 306 | result = iterate_mappings(query=query, idx=0, metrics=metrics) 307 | if not bool(result): 308 | fields_list.remove(_field) 309 | if not bool(fields_list): 310 | continue 311 | 312 | # validate first_metric_list 313 | if _node_check: 314 | _nodes = target[0][:-1][1:].split(',') 315 | else: 316 | _nodes = [target[0]] 317 | _nodes = set(first_metric_list) & set(_nodes) 318 | _nodes = ['.'.join(_.split('_')) for _ in _nodes] 319 | if not bool(_nodes): 320 | continue 321 | 322 | # Trigger rendering 323 | for _node in _nodes: 324 | for _doc_type in _doc_types: 325 | try: 326 | response = build_query(_doc_type, _node, fields_list, response, _COUNT, _FROM, _UNTIL, target) 327 | except: 328 | continue 329 | 330 | except: 331 | continue 332 | 333 | return json.dumps(response) 334 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Django settings for es-graphite-shim project 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.6/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.6/ref/settings/ 9 | """ 10 | 11 | # Quick-start development settings - unsuitable for production 12 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 13 | import os 14 | 15 | BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 16 | 17 | try: 18 | from local_settings import * 19 | except Exception as e: 20 | raise 21 | 22 | import socket 23 | hostname = socket.gethostname() 24 | del socket 25 | 26 | # SECURITY WARNING: don't run with debug turned on in production! 27 | if hostname == HOSTNAME: 28 | DEBUG = False 29 | PRODUCTION = True 30 | else: 31 | # All other environments are assumed to be non-production 32 | # environments. You can override this settings here to test out production 33 | # like behaviors. 34 | DEBUG = True 35 | PRODUCTION = False 36 | 37 | TEMPLATE_DEBUG = True 38 | 39 | # FIXME: If you want to test development mode with ./manage.py runserver as if 40 | # it would be production, you need to add 'localhost' or '*' to the 41 | # ALLOWED_HOSTS list, set DEBUG above to False, and you need to use the 42 | # --insecure switch on runserver (e.g. $ python3 ./manage.py runserver 43 | # --insecure). 44 | 45 | if DEBUG: 46 | ALLOWED_HOSTS = [''] 47 | elif PRODUCTION: 48 | # ALLOWED_HOSTS = [HOSTNAME,] 49 | ALLOWED_HOSTS = ['*'] 50 | else: 51 | ALLOWED_HOSTS = ['localhost',] 52 | 53 | # Application definition 54 | 55 | INSTALLED_APPS = ( 56 | 'django.contrib.admin', 57 | 'django.contrib.auth', 58 | 'django.contrib.contenttypes', 59 | 'django.contrib.sessions', 60 | 'django.contrib.messages', 61 | 'django.contrib.staticfiles', 62 | ) 63 | 64 | MIDDLEWARE_CLASSES = ( 65 | 'django.contrib.sessions.middleware.SessionMiddleware', 66 | 'django.middleware.common.CommonMiddleware', 67 | 'django.middleware.csrf.CsrfViewMiddleware', 68 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 69 | 'django.contrib.messages.middleware.MessageMiddleware', 70 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 71 | ) 72 | 73 | ROOT_URLCONF = 'es-graphite-shim.urls' 74 | WSGI_APPLICATION = 'es-graphite-shim.wsgi.application' 75 | SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' 76 | 77 | # Database 78 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 79 | 80 | DATABASES = {} 81 | # DATABASES = { 82 | # 'default': { 83 | # 'NAME': os.path.join(BASE_DIR, 'storage/%s.sqlite3' % (DB_NAME)), 84 | # 'ENGINE': 'django.db.backends.sqlite3', 85 | # 'USER': 'apache', 86 | # 'PASSWORD': DB_PASS, 87 | # 'HOST': '', 88 | # 'PORT': '' 89 | # } 90 | # } 91 | 92 | 93 | # Internationalization 94 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 95 | 96 | LANGUAGE_CODE = 'en-us' 97 | 98 | # FIXME: either remove, or configure based on local settings 99 | 100 | USE_I18N = True 101 | 102 | USE_L10N = True 103 | 104 | USE_TZ = True 105 | 106 | STATIC_ROOT = '/mnt/static' 107 | 108 | # Static files (CSS, JavaScript, Images) 109 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 110 | STATICFILES_DIRS = ( 111 | os.path.join(BASE_DIR, 'static'), 112 | ) 113 | 114 | STATIC_URL = '/static/' 115 | 116 | # List of finder classes that know how to find static files in 117 | # various locations. 118 | STATICFILES_FINDERS = ( 119 | 'django.contrib.staticfiles.finders.FileSystemFinder', 120 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 121 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 122 | ) 123 | 124 | TEMPLATE_LOADERS = ( 125 | 'django.template.loaders.filesystem.Loader', 126 | 'django.template.loaders.app_directories.Loader', 127 | # 'django.template.loaders.eggs.Loader', 128 | ) 129 | 130 | TEMPLATE_DIRS = (os.path.join(BASE_DIR, 'templates/'),) 131 | 132 | # Initiate ElasticSearch connection 133 | from elasticsearch import Elasticsearch #, client 134 | from urllib3 import Timeout 135 | timeoutobj = Timeout(total=1200, connect=10, read=600) 136 | 137 | from time import ctime 138 | print("[%s] - Initiating ES connection" % (ctime())) 139 | ES = Elasticsearch(host=ES_HOST, port=ES_PORT, 140 | timeout=timeoutobj, max_retries=0) 141 | print("[%s] - Established ES connection" % (ctime())) 142 | # get the data to be displayed in drop down list in grafana 143 | from lib.get_es_metadata import get_fieldnames as _get_fieldnames 144 | from lib.get_es_metadata import get_open_indices_list as _get_open_indices_list 145 | # query list of indices in state:open 146 | 147 | import json as _js 148 | _indices_path = os.path.join(BASE_DIR, 'lib/mappings/open_indices.json') 149 | 150 | try: 151 | if not os.path.exists(_indices_path): 152 | _OPEN_INDICES = _get_open_indices_list(ES, INDEX_PREFIX, DOC_TYPE) 153 | # dict with index name as key and fieldnames as values 154 | f = open(_indices_path, 'wb') 155 | f.write(bytes(_js.dumps(_OPEN_INDICES), 'UTF-8')) 156 | f.close() 157 | else: 158 | f = open(_indices_path, 'rb') 159 | _OPEN_INDICES = _js.loads(f.read().decode('UTF-8')) 160 | f.close() 161 | except Exception as e: 162 | quit("[%s] - ERROR: %s" % (ctime(), e)) 163 | 164 | print("[%s] - # of Open Indices: %d" % (ctime(), len(_OPEN_INDICES))) 165 | 166 | _FIELDS = _get_fieldnames(ES, FIELD, _OPEN_INDICES, doc_type=DOC_TYPE) 167 | # remove methods which won't be used any longer 168 | del _get_fieldnames 169 | del _get_open_indices_list 170 | 171 | 172 | # build an aggregate dict of mappings to be referred 173 | # for field validation each time a query is issued 174 | from lib.get_es_metadata import get_mappings as _get_mappings 175 | _MAPPINGS = _get_mappings(ES, DOC_TYPE, _fresh=FRESH) 176 | del _get_mappings 177 | del _js 178 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | urlpatterns = patterns('', 7 | url(r'^admin/', include(admin.site.urls)), 8 | ) 9 | urlpatterns += patterns('es-graphite-shim.views', 10 | url(r'^$', 'homepage', name='home'), 11 | url(r'^mapping/?$', 'view_mapping', name='mapping'), 12 | url(r'^render/?$', 'metrics_render', name='render'), 13 | url(r'^metrics/find/?$', 'metrics_find', name='find'), 14 | url(r'^dashboard/find/?$', 'dashboard_find', name='dash_find'), 15 | ) 16 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/views.py: -------------------------------------------------------------------------------- 1 | from django.template import RequestContext 2 | from django.http import HttpResponse, HttpResponseNotFound 3 | from django.views.decorators.csrf import csrf_exempt #csrf_protect 4 | from django.shortcuts import render_to_response 5 | from . import query_formatter as qf 6 | 7 | def homepage(request): 8 | """ 9 | Overview / Landing Page 10 | """ 11 | return render_to_response('index.html', 12 | { 'project_name' : 'Query Processor', 13 | 'title' : 'Query Middleware app', 14 | 'metric_path' : 'sample_metric.{param1,param2}', 15 | 'epoch_from' : 1411890859, 16 | 'epoch_until' : 1424890859, 17 | }) 18 | 19 | 20 | def view_mapping(request): 21 | """ 22 | List all available fields 23 | """ 24 | return HttpResponse(qf.json.dumps(qf.settings._MAPPINGS), 25 | content_type="application/json") 26 | 27 | 28 | @csrf_exempt 29 | def metrics_render(request): 30 | if request.method == 'POST': 31 | _TARGETS = request.POST.getlist('target', None) 32 | _FROM = request.POST.get('from', None) 33 | _UNTIL = request.POST.get('until', None) 34 | 35 | # _COUNT = request.POST.get('maxDataPoints', 10) 36 | # FIXME: limit count to 10 for now. Remove constraint 37 | # once caching and other optimization techniques are 38 | # included, so that ES instance doesn't get bombarded 39 | # every time maxDataPoints goes above 1000 or something 40 | _COUNT = 10 41 | 42 | _FORMAT = request.POST.get('format', None) 43 | 44 | if _TARGETS and _FROM and _UNTIL and (_FORMAT == 'json'): 45 | return HttpResponse(qf.render_metrics(_TARGETS, 46 | _FROM, 47 | _UNTIL, 48 | _COUNT), 49 | content_type="application/json") 50 | else: 51 | return HttpResponse('[]', content_type="application/json") 52 | 53 | elif request.method == 'GET': 54 | _TARGETS = request.GET.getlist('target', None) 55 | _FROM = request.GET.get('from', None) 56 | _UNTIL = request.GET.get('until', None) 57 | 58 | # _COUNT = request.GET.get('maxDataPoints', 10) 59 | _COUNT = 10 60 | 61 | _FORMAT = request.GET.get('format', None) 62 | if _TARGETS and _FROM and _UNTIL and (_FORMAT == 'json'): 63 | return HttpResponse(qf.render_metrics(_TARGETS, 64 | _FROM, 65 | _UNTIL, 66 | _COUNT), 67 | content_type="application/json") 68 | else: 69 | return HttpResponse('[]', content_type="application/json") 70 | 71 | else: 72 | return HttpResponseNotFound('

Method Not Allowed

') 73 | 74 | 75 | def metrics_find(request): 76 | """ 77 | Query for metric names, for templates and drop down list selection 78 | """ 79 | if request.method == 'GET': 80 | try: 81 | query = request.GET.get('query', None) 82 | if query: 83 | return HttpResponse(qf.find_metrics(query), 84 | content_type="application/json") 85 | else: 86 | return HttpResponse("Missing required parameter 'query'") 87 | except Exception as e: 88 | return HttpResponse('[]', content_type="application/json") 89 | 90 | else: 91 | return HttpResponseNotFound('

Method Not Allowed

') 92 | 93 | 94 | def dashboard_find(request): 95 | """ 96 | Query for metric names, for templates and drop down list selection 97 | """ 98 | # FIXME: understand this query, for proper response 99 | if request.method == 'GET': 100 | # query = request.GET.get('query', None) 101 | resp = { "dashboards": [ ] } 102 | return HttpResponse(qf.json.dumps(resp), content_type="application/json") 103 | 104 | else: 105 | return HttpResponseNotFound('

Method Not Allowed

') 106 | -------------------------------------------------------------------------------- /es-graphite-shim/es-graphite-shim/wsgi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | WSGI config for es-graphite-shim project. 4 | 5 | It exposes the WSGI callable as a module-level variable named ``application``. 6 | 7 | For more information on this file, see 8 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 9 | """ 10 | 11 | import os 12 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "es-graphite-shim.settings") 13 | 14 | from django.core.wsgi import get_wsgi_application 15 | application = get_wsgi_application() 16 | -------------------------------------------------------------------------------- /es-graphite-shim/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/lib/__init__.py -------------------------------------------------------------------------------- /es-graphite-shim/lib/get_es_metadata.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from elasticsearch import client 4 | from collections import defaultdict 5 | from time import ctime 6 | 7 | BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 8 | 9 | # def get_fieldnames(es_conn, _field, _indices, result=defaultdict(list), doc_type=''): 10 | def get_fieldnames(es_conn, _field, _indices, result=[], doc_type=''): 11 | """ 12 | Runs Elasticsearch aggs for all fieldnames 13 | as per value of _field 14 | """ 15 | _aggs = { 'aggs': {'aggs_custom': {'terms': {'field': _field, 16 | 'order': 17 | {'_term':'asc'}, 18 | 'size': 10000 }}}, 19 | 'fields': [] } 20 | 21 | N = 50 22 | for _ix in range(0, len(_indices), N): 23 | print("[%s] - %d Indices being processed starting from %d" % (ctime(), N, _ix)) 24 | current = [] 25 | res = es_conn.search(doc_type=doc_type, body=_aggs, 26 | search_type='count', index=_indices[_ix:_ix+N]) 27 | 28 | # convert '.' separated values to '_' separated 29 | # because grafana treats '.' separated as metric name 30 | for item in res['aggregations']['aggs_custom']['buckets']: 31 | current.append('_'.join(item['key'].split('.'))) 32 | 33 | result.extend(list(set(current))) 34 | 35 | return result 36 | 37 | 38 | def issue_mappings_query(es_conn, doc_type): 39 | """ 40 | Issue actual ES query to retrieve mappings 41 | """ 42 | es_client = client.IndicesClient(es_conn) 43 | _MAPPINGS = es_client.get_mapping(doc_type=doc_type) 44 | d = {} 45 | for i in list(_MAPPINGS.values()): 46 | d.update(i) 47 | 48 | return d 49 | 50 | 51 | def get_open_indices_list(es_conn, prefix, doc_type): 52 | """ 53 | Query list of all open indices and return list. 54 | """ 55 | es_client = client.ClusterClient(es_conn) 56 | response = es_client.state(ignore_unavailable=True, metric="metadata") 57 | resp = [ix for ix in list(response['metadata']['indices'].keys()) \ 58 | if '%s.%s' % (prefix, doc_type) in ix] 59 | return resp 60 | 61 | 62 | def get_mappings(es_conn, doc_type, _fresh=False): 63 | """ 64 | Return one single mapping combined by combining 65 | individual index mappings to form a big updated list. 66 | 67 | This returns a saved mapping from a json file or it 68 | may issue an actual query to ES based on value of _fresh 69 | """ 70 | if _fresh == 'True': 71 | return issue_mappings_query(es_conn, doc_type) 72 | else: 73 | try: 74 | f = open(os.path.join(BASE_DIR, 75 | 'lib/mappings/es_mappings.json'), 'rb') 76 | _MAPPINGS = json.loads(f.read().decode('utf-8')) 77 | f.close() 78 | return _MAPPINGS 79 | except: 80 | print("[%s] - WARNING: es_mappings.json doesn't exist; falling back to issuing es query"\ 81 | % (ctime())) 82 | return issue_mappings_query(es_conn, doc_type) 83 | -------------------------------------------------------------------------------- /es-graphite-shim/lib/mappings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/lib/mappings/__init__.py -------------------------------------------------------------------------------- /es-graphite-shim/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "es-graphite-shim.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /es-graphite-shim/storage/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/storage/.gitkeep -------------------------------------------------------------------------------- /es-graphite-shim/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/storage/__init__.py -------------------------------------------------------------------------------- /es-graphite-shim/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | {% block title %}{{ project_name }} - Page Not Found{% endblock %} 4 | 5 | {% block error %} 6 |
7 |
8 |

{{ project_name }} - Page Not Found

9 |
10 |
11 |

We are unable to find the page you requested.

12 |
13 |
14 | {% endblock %} 15 | -------------------------------------------------------------------------------- /es-graphite-shim/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | {% block title %}{{title}}{% endblock %} 4 | 5 | {% block error %} 6 |
7 |
8 |

{{ project_name }} - Internal Server Error

9 |
10 |
11 |

We encountered an unexpected error processing your 12 | {% if error.name %} {{ error.name }}{% endif %}{% if error %}: 13 | {% if error.msg %}"{{ error.msg }}"{% else %}"{{ error }}"{% endif %}{% endif %}. 14 | Please try again later.

15 |
16 |
17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /es-graphite-shim/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/distributed-system-analysis/es-graphite-shim/bd462fcd530b48be602cf5f243f6792502d81be8/es-graphite-shim/templates/__init__.py -------------------------------------------------------------------------------- /es-graphite-shim/templates/base.html: -------------------------------------------------------------------------------- 1 | {% load staticfiles %}{% load static from staticfiles %} 2 | 3 | 4 | 5 | 6 | {% block title %}{{ title }}{% endblock %} 7 | 8 | 9 |
10 |
11 | {{ project_name }} 12 |
13 | {% block content %}{% endblock %} 14 |
15 | 16 | {% block error %}{% endblock %} 17 | 18 | 19 | -------------------------------------------------------------------------------- /es-graphite-shim/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %}{% load staticfiles %}{% load static from staticfiles %} 2 | 3 | {% block content %} 4 |
5 |
    6 |
  1. Get Mapping for all ES indices
    2. 7 |
  2. Render Data for this query (click link to check out metric pattern)
    3. 8 |
  3. Query all first metric types in DOT separated metric path for all ES indices
    4. 9 |
10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.8.2 2 | elasticsearch==1.6.0 3 | pytest==2.7.1 4 | requests>=2.20.0 5 | --------------------------------------------------------------------------------