├── .circleci └── config.yml ├── .gitignore ├── Dockerfile ├── GrafanaDatastoreServer.py ├── LICENSE ├── README.md ├── compose ├── docker-compose.yml ├── example │ ├── README.md │ ├── docker-compose.yml │ └── producer │ │ ├── Dockerfile │ │ ├── producer.py │ │ └── requirements.txt └── grafana │ ├── Dockerfile │ ├── config.ini │ └── provisioning │ └── datasources │ └── redistimeseries.yaml ├── requirements.txt └── test.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | 2 | # Python CircleCI 2.0 configuration file 3 | # 4 | # Check https://circleci.com/docs/2.0/language-python/ for more details 5 | # 6 | version: 2 7 | jobs: 8 | build: 9 | docker: 10 | - image: circleci/python:3.7.1 11 | - image: redislabs/redistimeseries:latest 12 | 13 | working_directory: ~/repo 14 | 15 | steps: 16 | - checkout 17 | 18 | - restore_cache: # Download and cache dependencies 19 | keys: 20 | - v1-dependencies-{{ checksum "requirements.txt" }} 21 | # fallback to using the latest cache if no exact match is found 22 | - v1-dependencies- 23 | 24 | - run: 25 | name: install dependencies 26 | command: | 27 | virtualenv venv 28 | . venv/bin/activate 29 | pip install -r requirements.txt 30 | pip install codecov 31 | 32 | - save_cache: 33 | paths: 34 | - ./venv 35 | key: v1-dependencies-{{ checksum "requirements.txt" }} 36 | 37 | - run: 38 | name: run tests 39 | command: | 40 | . venv/bin/activate 41 | REDIS_PORT=6379 coverage run test.py 42 | codecov 43 | 44 | workflows: 45 | version: 2 46 | commit: 47 | jobs: 48 | - build 49 | nightly: 50 | triggers: 51 | - schedule: 52 | cron: "0 0 * * *" 53 | filters: 54 | branches: 55 | only: 56 | - master 57 | jobs: 58 | - build 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # eclipse 107 | .project 108 | .pydevproject 109 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | 4 | WORKDIR /app/ 5 | 6 | COPY requirements.txt /app/ 7 | 8 | RUN apk update && apk add --no-cache --virtual build-deps build-base gcc musl-dev libffi-dev libev-dev 9 | RUN pip install -r requirements.txt 10 | RUN apk del build-deps 11 | 12 | COPY GrafanaDatastoreServer.py /app/ 13 | 14 | EXPOSE 8080 15 | ENV REDIS_HOST=localhost 16 | ENV REDIS_PORT=6379 17 | 18 | CMD /app/GrafanaDatastoreServer.py --redis-server $REDIS_HOST --redis-port $REDIS_PORT 19 | -------------------------------------------------------------------------------- /GrafanaDatastoreServer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import redis 5 | import flask 6 | from datetime import timedelta, datetime 7 | import dateutil.parser 8 | from gevent.pywsgi import WSGIServer 9 | from flask import Flask, jsonify 10 | from flask_cors import CORS, cross_origin 11 | 12 | app = Flask(__name__) 13 | CORS(app) 14 | 15 | REDIS_POOL = None 16 | SCAN_TYPE_SCRIPT = """local cursor, pat, typ, cnt = ARGV[1], ARGV[2], ARGV[3], ARGV[4] or 100 17 | local rep = {} 18 | 19 | local res = redis.call('SCAN', cursor, 'MATCH', pat, 'COUNT', cnt) 20 | while #res[2] > 0 do 21 | local k = table.remove(res[2]) 22 | local t = redis.call('TYPE', k) 23 | if t['ok'] == typ then 24 | table.insert(rep, k) 25 | end 26 | end 27 | 28 | rep = {tonumber(res[1]), rep} 29 | return rep""" 30 | 31 | @app.route('/') 32 | @cross_origin() 33 | def hello_world(): 34 | return 'OK' 35 | 36 | @app.route('/search', methods=["POST", 'GET']) 37 | @cross_origin() 38 | def search(): 39 | redis_client = redis.Redis(connection_pool=REDIS_POOL) 40 | result = [] 41 | cursor = 0 42 | while True: 43 | cursor, keys = redis_client.eval(SCAN_TYPE_SCRIPT, 0, cursor, "*", "TSDB-TYPE", 100) 44 | result.extend([k.decode("ascii") for k in keys]) 45 | if cursor == 0: 46 | break 47 | 48 | return jsonify(result) 49 | 50 | def process_targets(targets, redis_client): 51 | result = [] 52 | for target in targets: 53 | if '*' in target: 54 | result.extend([k.decode('ascii') for k in redis_client.keys(target)]) 55 | else: 56 | result.append(target) 57 | return result 58 | 59 | @app.route('/query', methods=["POST", 'GET']) 60 | def query(): 61 | request = flask.request.get_json() 62 | response = [] 63 | 64 | # dates 'from' and 'to' are expected to be in UTC, which is what Grafana provides here. 65 | stime = dateutil.parser.parse(request['range']['from']).timestamp() * 1000 66 | etime = dateutil.parser.parse(request['range']['to']).timestamp() * 1000 67 | 68 | redis_client = redis.Redis(connection_pool=REDIS_POOL) 69 | targets = process_targets([t['target'] for t in request['targets']], redis_client) 70 | 71 | for target in targets: 72 | args = ['ts.range', target, int(stime), int(etime)] 73 | if 'intervalMs' in request and request['intervalMs'] > 0: 74 | args += ['avg', int(request['intervalMs'])] 75 | print(args) 76 | redis_resp = redis_client.execute_command(*args) 77 | datapoints = [(float(x2.decode("ascii")), x1) for x1, x2 in redis_resp] 78 | response.append(dict(target=target, datapoints=datapoints)) 79 | return jsonify(response) 80 | 81 | 82 | @app.route('/annotations') 83 | def annotations(): 84 | return jsonify([]) 85 | 86 | def main(): 87 | global REDIS_POOL 88 | parser = argparse.ArgumentParser() 89 | parser.add_argument("--host", help="server address to listen to", default="0.0.0.0") 90 | parser.add_argument("--port", help="port number to listen to", default=8080, type=int) 91 | parser.add_argument("--redis-server", help="redis server address", default="localhost") 92 | parser.add_argument("--redis-port", help="redis server port", default=6379, type=int) 93 | args = parser.parse_args() 94 | 95 | REDIS_POOL = redis.ConnectionPool(host=args.redis_server, port=args.redis_port) 96 | 97 | http_server = WSGIServer(('', args.port), app) 98 | http_server.serve_forever() 99 | 100 | if __name__ == '__main__': 101 | main() 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019, 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![license](https://img.shields.io/github/license/RedisTimeSeries/grafana-redistimeseries.svg)](https://github.com/RedisTimeSeries/grafana-redistimeseries) 2 | [![CircleCI](https://circleci.com/gh/RedisTimeSeries/grafana-redistimeseries/tree/master.svg?style=svg)](https://circleci.com/gh/RedisTimeSeries/grafana-redistimeseries/tree/master) 3 | [![GitHub issues](https://img.shields.io/github/release/RedisTimeSeries/grafana-redistimeseries.svg)](https://github.com/RedisTimeSeries/grafana-redistimeseries/releases/latest) 4 | [![Codecov](https://codecov.io/gh/RedisTimeSeries/grafana-redistimeseries/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisTimeSeries/grafana-redistimeseries) 5 | 6 | # Deprecation notice 7 | This project has been deprecated, for further information on how to use Redis and Grafana: https://github.com/RedisTimeSeries/grafana-redis-datasource. 8 | 9 | # RedisTimeSeries-Datasource 10 | [![Forum](https://img.shields.io/badge/Forum-RedisTimeSeries-blue)](https://forum.redislabs.com/c/modules/redistimeseries) 11 | [![Gitter](https://badges.gitter.im/RedisLabs/RedisTimeSeries.svg)](https://gitter.im/RedisLabs/RedisTimeSeries?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) 12 | 13 | Grafana Datasource for RedisTimeSeries 14 | 15 | ## QuickStart 16 | You can tryout the `Grafana Datasource for RedisTimeSeries` with RedisTimeSeries and Grafana in a single docker compose 17 | ```bash 18 | cd compose 19 | docker-compose up 20 | ``` 21 | Grafana can be accessed on port 3000 (admin:admin) 22 | 23 | ## Grafana Datastore API Server 24 | ### Overview 25 | A HTTP Server to serve metrics to Grafana via the simple-json-datasource 26 | 27 | ### Grafana configuration 28 | 29 | 1. install SimpleJson data source: https://grafana.net/plugins/grafana-simple-json-datasource/installation 30 | 2. in Grafana UI, go to Data Sources 31 | 3. Click `Add data source` 32 | 3.1 choose Name 33 | 3.2 Type: `SimpleJson` 34 | 3.3 URL: point to the URL for your GrafanaDatastoreServer.py 35 | 3.4 Access: direct (unless you are using a proxy) 36 | 37 | 4. Query the datasource by a specific key, or * for a wildcard, for example: `stats_counts.http.*` 38 | 39 | ### Dependencies 40 | To install the needed dependencies just run: pip install -r requirements.txt 41 | 42 | ### GrafanaDatastoreServer.py Usage 43 | ``` 44 | usage: GrafanaDatastoreServer.py [-h] [--host HOST] [--port PORT] 45 | [--redis-server REDIS_SERVER] 46 | [--redis-port REDIS_PORT] 47 | 48 | optional arguments: 49 | -h, --help show this help message and exit 50 | --host HOST server address to listen to 51 | --port PORT port number to listen to 52 | --redis-server REDIS_SERVER 53 | redis server address 54 | --redis-port REDIS_PORT 55 | redis server port 56 | ``` 57 | 58 | #### Note about timezone 59 | Grafana uses UTC timestamps to query its datastores. This datastore will use the same timestamps to query Redis, which means that it assumes all timestamps are UTC based. 60 | -------------------------------------------------------------------------------- /compose/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | redis: 4 | image: "redislabs/redistimeseries:edge" 5 | ports: 6 | - "6379:6379" 7 | grafana_redis_source: 8 | build: ../. 9 | environment: 10 | - "REDIS_HOST=redis" 11 | depends_on: 12 | - redis 13 | grafana: 14 | build: ./grafana/ 15 | ports: 16 | - "3000:3000" 17 | depends_on: 18 | - grafana_redis_source 19 | -------------------------------------------------------------------------------- /compose/example/README.md: -------------------------------------------------------------------------------- 1 | # RedisTimeSeries + Grafana + Grafana Redis source 2 | 3 | ## Run the stack 4 | 5 | ```bash 6 | docker-compose up -d 7 | ``` 8 | 9 | ## Setup Grafana 10 | 11 | open the browser to url: 12 | 13 | [http://localhost:3000](http://localhost:3000) 14 | 15 | Default username and passwords are: 16 | - username: `admin` 17 | - password: `admin` 18 | 19 | ## Setup the data source in Grafana 20 | 21 | - On dashboard, click `Add Data Source` 22 | 23 | - Select `SimpleJson` datasource 24 | 25 | - Name the source a recognizable name, like `RedisTimeSeries` 26 | 27 | - In URL: enter `http://grafana_redis_source:8080` (the internal name for the service, exposed at port 8080) 28 | 29 | - Click `Test and Save` 30 | 31 | - It should show a green notification stating the connection worked. 32 | 33 | ## Create a Dashboard 34 | 35 | - Go to Dashboards -> Home 36 | 37 | - Click `New Dashboard` 38 | 39 | - Click `Add Query` 40 | 41 | - In Query Select the `RedisTimeSeries` data source you created earlier 42 | 43 | - In the Query `timeserie` dropdown enter `temperature` which is the name of the key produced by the producer example. 44 | 45 | - You should now see some data in the chart. 46 | 47 | - Save the Dashboard with the top right Save icon (note this is saved in the container so config will be lost if you delete the container) 48 | In order to save config, modify the `docker-compose.yml` file to mount a storage volume. 49 | 50 | -------------------------------------------------------------------------------- /compose/example/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | 4 | redis: 5 | image: redislabs/redistimeseries 6 | networks: 7 | - ts 8 | 9 | grafana: 10 | build: 11 | context: ../grafana 12 | dockerfile: Dockerfile 13 | ports: 14 | - "127.0.0.1:3000:3000" 15 | networks: 16 | - ts 17 | 18 | grafana_redis_source: 19 | build: 20 | context: ../../ 21 | dockerfile: Dockerfile 22 | environment: 23 | - "REDIS_HOST=redis" 24 | depends_on: 25 | - redis 26 | - grafana 27 | networks: 28 | - ts 29 | 30 | producer: 31 | restart: always 32 | build: 33 | context: producer 34 | dockerfile: Dockerfile 35 | environment: 36 | - "REDIS_HOST=redis" 37 | depends_on: 38 | - redis 39 | - grafana_redis_source 40 | networks: 41 | - ts 42 | 43 | networks: 44 | ts: 45 | driver: bridge -------------------------------------------------------------------------------- /compose/example/producer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine3.9 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt /app/ 6 | 7 | RUN pip install -r requirements.txt 8 | 9 | COPY producer.py /app/ 10 | 11 | ENV REDIS_HOST=localhost 12 | ENV REDIS_PORT=6379 13 | 14 | CMD /app/producer.py --redis-server $REDIS_HOST --redis-port $REDIS_PORT 15 | -------------------------------------------------------------------------------- /compose/example/producer/producer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import math 5 | import random 6 | from time import sleep 7 | from redistimeseries.client import Client as RedisTimeSeries 8 | 9 | 10 | def main(): 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument("--redis-server", help="redis server address", default="localhost") 13 | parser.add_argument("--redis-port", help="redis server port", default=6379, type=int) 14 | args = parser.parse_args() 15 | print(""" 16 | Starting with: 17 | redis server: {} 18 | redis port: {} 19 | """.format(args.redis_server, args.redis_port)) 20 | 21 | rts = RedisTimeSeries(port=args.redis_port, host=args.redis_server) 22 | 23 | try: 24 | rts.create('temperature', retentionSecs=60*24, labels={'sensorId': '2'}) 25 | except Exception as e: 26 | # will except if key already exists (i.e. on restart) 27 | print(str(e)) 28 | 29 | variance = 0 30 | t = 0 31 | while True: 32 | # add with current timestamp 33 | print(".", end="") 34 | variance += (random.random() - 0.5) / 10.0 35 | t += 1 36 | value = math.cos(t / 100) + variance 37 | rts.add('temperature', '*', value) 38 | sleep(0.1) 39 | 40 | 41 | if __name__ == '__main__': 42 | main() 43 | -------------------------------------------------------------------------------- /compose/example/producer/requirements.txt: -------------------------------------------------------------------------------- 1 | redistimeseries==0.2 2 | six==1.12.0 -------------------------------------------------------------------------------- /compose/grafana/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM grafana/grafana 2 | 3 | RUN grafana-cli plugins install grafana-simple-json-datasource 4 | 5 | ADD ./config.ini /etc/grafana/config.ini 6 | ADD ./provisioning/datasources/redistimeseries.yaml /etc/grafana/provisioning/datasources/redistimeseries.yaml 7 | -------------------------------------------------------------------------------- /compose/grafana/config.ini: -------------------------------------------------------------------------------- 1 | [paths] 2 | provisioning = /etc/grafana/provisioning 3 | 4 | [server] 5 | enable_gzip = true 6 | -------------------------------------------------------------------------------- /compose/grafana/provisioning/datasources/redistimeseries.yaml: -------------------------------------------------------------------------------- 1 | # config file version 2 | apiVersion: 1 3 | 4 | datasources: 5 | - name: RedisTimeSeries 6 | type: grafana-simple-json-datasource 7 | access: proxy 8 | orgId: 1 9 | url: http://grafana_redis_source:8080 10 | isDefault: true 11 | version: 1 12 | editable: true 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | redis==2.10.5 2 | gevent>=1.3.a1 3 | Flask>=0.12 4 | Flask-Cors==3.0.2 5 | python-dateutil==2.6.0 6 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RedisTimeSeries/grafana-redistimeseries/49c777c683862a08e2176be2330cf4cfea168fc3/test.py --------------------------------------------------------------------------------