├── .gitignore ├── CHANGELOG.md ├── Dockerfile.example ├── Dockerfile.tests ├── LICENSE ├── README.md ├── django_example ├── django_example │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ ├── views.py │ └── wsgi.py ├── manage.py ├── metrics │ ├── __init__.py │ ├── apps.py │ ├── general.py │ ├── middleware.py │ └── view.py └── requirements.txt ├── example-docker-compose.yml ├── prometheus_redis_client ├── __init__.py ├── base_metric.py ├── helpers.py ├── metrics.py └── registry.py ├── requirements.txt ├── setup.py ├── test-docker-compose.yml ├── tests ├── __init__.py ├── helpers.py ├── test_common_gauge.py ├── test_counters.py ├── test_gauge.py ├── test_histogram.py └── test_summary.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | .pytest_cache/ 2 | .git/ 3 | .idea/ 4 | __pycache__/ 5 | dist/ 6 | Pipfile 7 | Pipfile.lock 8 | MANIFEST 9 | *.pyc 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | #### 0.5.0 5 | 6 | * Add `set` to Counter (@darkman66) 7 | 8 | #### 0.4.0 9 | 10 | * Add `inc` and `dec` to CommonGauge (@maxfrei) 11 | 12 | #### 0.3.0 13 | 14 | * Fix documentation 15 | * Add expire for CommonGauge. 16 | 17 | #### 0.2.0 18 | 19 | * Add CommonGauge metric. 20 | * Add timeit decorator for Summary and Histogram 21 | * Start tests via tox in docker for python 3.5, 3.6, 3.7. 22 | 23 | 24 | #### 0.1.0 25 | 26 | * Metric Counter, Summary, Histogram and Gauge (per process). -------------------------------------------------------------------------------- /Dockerfile.example: -------------------------------------------------------------------------------- 1 | FROM python:3.7.3-stretch 2 | 3 | COPY django_example /src/ 4 | RUN pip install -r /src/requirements.txt 5 | 6 | COPY prometheus_redis_client /package/prometheus_redis_client/prometheus_redis_client 7 | COPY setup.py /package/prometheus_redis_client/setup.py 8 | WORKDIR /package/prometheus_redis_client/ 9 | RUN python setup.py sdist 10 | RUN pip install /package/prometheus_redis_client/dist/* 11 | 12 | WORKDIR /src/ 13 | 14 | CMD PROMETHEUS_REDIS_URI=redis://redis:6379 python manage.py runserver 0.0.0.0:8000 -------------------------------------------------------------------------------- /Dockerfile.tests: -------------------------------------------------------------------------------- 1 | FROM belousovalex/tox_image 2 | 3 | COPY requirements.txt . 4 | RUN pip3 install -r requirements.txt && pip3 install tox==3.9.0 5 | 6 | RUN mkdir /src/ 7 | 8 | WORKDIR /src/ 9 | 10 | COPY setup.py . 11 | COPY tox.ini . 12 | COPY prometheus_redis_client ./prometheus_redis_client/ 13 | COPY tests ./tests 14 | 15 | CMD tox -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2019 Belousov Aleskey 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Prometheus Redis client 2 | Python prometheus client that store metrics in Redis. 3 | 4 | Use it in multiprocessing applications (started via gunicorn\uwsgi) and in deferred tasks like a celery tasks. 5 | 6 | ### Installation 7 | 8 | $ pip install prometheus_redis_client 9 | 10 | Support Semver. Use >=0.x.0,<0.x+1.0 in you requirements file if you dont want to break code. 11 | 12 | ### Base usage 13 | 14 | You can make global variable and use it when you want change metrics value. Support four type of values: Counter, Histogram, Summary, Gauge. 15 | 16 | Each metric variable bind to some `prometheus_redis_client.Registry` object. By default its `prometheus_redis_client.REGISTRY`. 17 | Registry contains redis client. So you can store metric values in different Redis instance if you make you own Registry object and bind it with your metrics. 18 | 19 | ##### Setup REGISTRY 20 | 21 | import redis 22 | from prometheus_redis_client import REGISTRY 23 | 24 | REGISTRY.set_redis(redis.from_url("redis://localhost:6379")) 25 | 26 | ##### Counter 27 | 28 | from prometheus_redis_client import Counter 29 | simple_counter = Counter('simple_counter', 'Simple counter') 30 | counter_with_labels = Counter( 31 | 'counter_with_labels', 32 | 'Counter with labels', 33 | labelnames=["name"], 34 | ) 35 | 36 | * increase counter 37 | 38 | def some_function(): 39 | ... 40 | simple_cointer.inc() 41 | counter_with_labels.labels(name="piter").inc(2) 42 | ... 43 | 44 | 45 | * support for set current value 46 | 47 | def some_function(): 48 | ... 49 | simple_cointer.set(100) 50 | counter_with_labels.labels(name="piter").set(2) 51 | ... 52 | 53 | ##### Summary 54 | 55 | from prometheus_redis_client import Summary 56 | simple_summary = Summary('simple_summary', 'Simple summary') 57 | summary_with_labels = Summary( 58 | 'summary_with_labels', 59 | 'Summary with labels', 60 | labelnames=["name"], 61 | ) 62 | 63 | def some_function(): 64 | ... 65 | simple_summary.inc() 66 | summary_with_labels.labels(name="piter").inc(2) 67 | ... 68 | 69 | You can use decorator for time some function processing. 70 | 71 | @simple_summary.timeit() 72 | def another_func(): 73 | ... 74 | 75 | # and with labels 76 | @summary_with_labels.timeit(name='greg') 77 | def another_func2(): 78 | ... 79 | 80 | ##### Histogram 81 | 82 | from prometheus_redis_client import Histogram 83 | simple_histogram = Histogram('simple_histogram', 'Simple histogram') 84 | histogram_with_labels = Histogram( 85 | 'histogram_with_labels', 86 | 'Histogram with labels', 87 | labelnames=["name"], 88 | ) 89 | 90 | def some_function(): 91 | ... 92 | simple_histogram.observe(2.34) 93 | histogram_with_labels.labels(name="piter").observe(0.43) 94 | ... 95 | 96 | You can use decorator for time function. 97 | 98 | @simple_histogram.timeit() 99 | def another_func(): 100 | ... 101 | 102 | # and with labels 103 | @histogram_with_labels.timeit(name='greg') 104 | def another_func2(): 105 | ... 106 | 107 | 108 | ##### CommonGauge 109 | 110 | CommonGauge its simple metric that set, increment or decrement value to same Redis key from any process. 111 | 112 | Represent as Gauge metric in output for Prometheus. 113 | 114 | from prometheus_redis_client import CommonGauge 115 | common_gauge = CommonGauge('common_gauge', 'Common gauge') 116 | gauge_with_labels = CommonGauge( 117 | 'common_gauge_with_labels', 118 | 'Common gauge with labels', 119 | labelnames=["name"], 120 | ) 121 | gauge_with_labels_and_expire = CommonGauge( 122 | 'common_gauge_with_labels', 123 | 'Common gauge with labels', 124 | labelnames=["name"], 125 | expire=2, # if you make set() then after 2 seconds Redis delete key/value for metric with given labels 126 | ) 127 | 128 | def some_function(): 129 | ... 130 | simple_gauge.set(2.34) 131 | gauge_with_labels.labels(name="piter").set(0.43) 132 | ... 133 | 134 | ##### Gauge 135 | 136 | Gauge metric per process. Add `gauge_index` label to metric as process identifier. 137 | 138 | from prometheus_redis_client import Gauge 139 | simple_gauge = Gauge('simple_gauge', 'Simple gauge') 140 | gauge_with_labels = Gauge( 141 | 'gauge_with_labels', 142 | 'gauge with labels', 143 | labelnames=["name"], 144 | ) 145 | 146 | def some_function(): 147 | ... 148 | simple_gauge.set(2.34) 149 | gauge_with_labels.labels(name="piter").set(0.43) 150 | ... 151 | 152 | Only Gauge metric set per process. 153 | If your application start via gunicorn\uwsgi with concurrency = N then you get N metrics value for each process. 154 | Metrics value contains `gauge_index` - its simple counter based on Redis. 155 | 156 | Gauge metrics set value in Redis with expire period because you application can restart after N requests (harakiry mode for example) or server shout down. 157 | 158 | Then after expire period gauge of dead process will be remove from metrics. 159 | 160 | But you can change gauge metrics less often then expire period set. 161 | For not to lose metrics value special thread will refresh gauge values with period less then expire timeout. 162 | 163 | 164 | ##### Export metrics 165 | 166 | You cat export metrics to text. Example: 167 | 168 | from prometheus_redis_client import REGISTRY 169 | REGISTRY.output() 170 | 171 | 172 | ### Contribution 173 | 174 | Welcome for contribution. 175 | 176 | Run tests if you have docker: 177 | 178 | $ docker-compose -f test-docker-compose.yml up --build 179 | 180 | Start django app example: 181 | 182 | $ docker-compose -f example-docker-compose.yml up --build 183 | 184 | -------------------------------------------------------------------------------- /django_example/django_example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belousovalex/prometheus_redis_client/90852fb0eaf3aee1937a74cad6181c304dc6999a/django_example/django_example/__init__.py -------------------------------------------------------------------------------- /django_example/django_example/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for django_example project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.2. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.2/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = '&0^mo=l5ubah9)$3u3f)9127*4up20dyg5ts^uccu$b7gfxxb*' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'metrics.apps.MetricAppConfig', 41 | ] 42 | 43 | MIDDLEWARE = [ 44 | 'metrics.middleware.MetricsMiddleware', 45 | 'django.middleware.security.SecurityMiddleware', 46 | 'django.contrib.sessions.middleware.SessionMiddleware', 47 | 'django.middleware.common.CommonMiddleware', 48 | 'django.middleware.csrf.CsrfViewMiddleware', 49 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 52 | ] 53 | 54 | ROOT_URLCONF = 'django_example.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'django_example.wsgi.application' 73 | 74 | PROMETHEUS_REDIS_URI = os.environ.get("PROMETHEUS_REDIS_URI", "redis://redis:6379") 75 | 76 | 77 | # Database 78 | # https://docs.djangoproject.com/en/2.2/ref/settings/#databases 79 | 80 | DATABASES = {} 81 | 82 | 83 | # Password validation 84 | # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators 85 | 86 | AUTH_PASSWORD_VALIDATORS = [ 87 | { 88 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 89 | }, 90 | { 91 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 92 | }, 93 | { 94 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 95 | }, 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 98 | }, 99 | ] 100 | 101 | 102 | # Internationalization 103 | # https://docs.djangoproject.com/en/2.2/topics/i18n/ 104 | 105 | LANGUAGE_CODE = 'en-us' 106 | 107 | TIME_ZONE = 'UTC' 108 | 109 | USE_I18N = True 110 | 111 | USE_L10N = True 112 | 113 | USE_TZ = True 114 | 115 | 116 | # Static files (CSS, JavaScript, Images) 117 | # https://docs.djangoproject.com/en/2.2/howto/static-files/ 118 | 119 | STATIC_URL = '/static/' 120 | -------------------------------------------------------------------------------- /django_example/django_example/urls.py: -------------------------------------------------------------------------------- 1 | """django_example URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.2/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.urls import path 17 | from django_example.views import increment_view 18 | from metrics.view import metrics_view 19 | 20 | urlpatterns = [ 21 | path('', increment_view, name="main"), 22 | path('metrics', metrics_view, name="metrics"), 23 | ] 24 | -------------------------------------------------------------------------------- /django_example/django_example/views.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | from django.http.response import HttpResponse 4 | 5 | 6 | def increment_view(request): 7 | sleep_time = random.randint(1, 700) / 1000.0 8 | time.sleep(sleep_time) 9 | return HttpResponse("%f" % sleep_time) 10 | -------------------------------------------------------------------------------- /django_example/django_example/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for django_example project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_example.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /django_example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | from pathlib import Path 7 | 8 | prometheus_redis_client_path = Path(__file__).absolute().parent.parent 9 | 10 | sys.path.insert(0, str(prometheus_redis_client_path)) 11 | 12 | 13 | def main(): 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_example.settings') 15 | try: 16 | from django.core.management import execute_from_command_line 17 | except ImportError as exc: 18 | raise ImportError( 19 | "Couldn't import Django. Are you sure it's installed and " 20 | "available on your PYTHONPATH environment variable? Did you " 21 | "forget to activate a virtual environment?" 22 | ) from exc 23 | execute_from_command_line(sys.argv) 24 | 25 | 26 | if __name__ == '__main__': 27 | main() 28 | -------------------------------------------------------------------------------- /django_example/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belousovalex/prometheus_redis_client/90852fb0eaf3aee1937a74cad6181c304dc6999a/django_example/metrics/__init__.py -------------------------------------------------------------------------------- /django_example/metrics/apps.py: -------------------------------------------------------------------------------- 1 | import redis 2 | from django.apps import AppConfig 3 | from django.conf import settings 4 | 5 | from prometheus_redis_client import REGISTRY 6 | 7 | 8 | class MetricAppConfig(AppConfig): 9 | name = 'metrics' 10 | verbose_name = "metrics" 11 | 12 | def ready(self): 13 | super().ready() 14 | REGISTRY.set_redis(redis.from_url(settings.PROMETHEUS_REDIS_URI)) 15 | 16 | -------------------------------------------------------------------------------- /django_example/metrics/general.py: -------------------------------------------------------------------------------- 1 | from prometheus_redis_client import Counter, Histogram 2 | 3 | count_of_requests = Counter( 4 | "count_of_requests", 5 | "Count of requests", 6 | labelnames=["viewname", ], 7 | ) 8 | 9 | request_latency = Histogram( 10 | "request_latency", 11 | "Request latency", 12 | labelnames=["viewname", ], 13 | buckets=[0.10, 0.50, 0.100, 0.500], 14 | ) -------------------------------------------------------------------------------- /django_example/metrics/middleware.py: -------------------------------------------------------------------------------- 1 | import time 2 | from metrics import general 3 | from django.urls import resolve 4 | 5 | 6 | class MetricsMiddleware(object): 7 | 8 | def __init__(self, get_response): 9 | self.get_response = get_response 10 | 11 | def __call__(self, request): 12 | viewname = resolve(request.path).view_name 13 | general.count_of_requests.labels(viewname=viewname).inc() 14 | start_time = time.time() 15 | try: 16 | return self.get_response(request) 17 | finally: 18 | general.request_latency.labels(viewname=viewname).observe(time.time() - start_time) -------------------------------------------------------------------------------- /django_example/metrics/view.py: -------------------------------------------------------------------------------- 1 | from django.http.response import HttpResponse 2 | from prometheus_redis_client import REGISTRY 3 | 4 | 5 | def metrics_view(request): 6 | return HttpResponse(REGISTRY.output(), content_type="text/plain") 7 | -------------------------------------------------------------------------------- /django_example/requirements.txt: -------------------------------------------------------------------------------- 1 | django==2.2.4 -------------------------------------------------------------------------------- /example-docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | redis: 4 | image: redis:latest 5 | example: 6 | build: 7 | context: . 8 | dockerfile: Dockerfile.example 9 | ports: 10 | - 8000:8000 11 | depends_on: 12 | - redis -------------------------------------------------------------------------------- /prometheus_redis_client/__init__.py: -------------------------------------------------------------------------------- 1 | from prometheus_redis_client.registry import REGISTRY, Registry, Refresher 2 | from prometheus_redis_client.metrics import CommonGauge, Counter, Gauge, Histogram, Summary, DEFAULT_GAUGE_INDEX_KEY -------------------------------------------------------------------------------- /prometheus_redis_client/base_metric.py: -------------------------------------------------------------------------------- 1 | """Module provide base Metric classes.""" 2 | import json 3 | import base64 4 | import logging 5 | from typing import List 6 | from functools import partial, wraps 7 | 8 | from prometheus_redis_client.registry import Registry, REGISTRY 9 | 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class BaseRepresentation(object): 15 | 16 | def output(self) -> str: 17 | raise NotImplementedError 18 | 19 | 20 | class MetricRepresentation(BaseRepresentation): 21 | 22 | def __init__(self, name, labels, value): 23 | self.name = name 24 | self.labels = labels 25 | self.value = value 26 | 27 | def output(self) -> str: 28 | if self.labels is None: 29 | labels_str = "" 30 | else: 31 | labels_str = ",".join([ 32 | '{key}=\"{val}\"'.format( 33 | key=key, 34 | val=self.labels[key] 35 | ) for key in sorted(self.labels.keys()) 36 | ]) 37 | if labels_str: 38 | labels_str = "{" + labels_str + "}" 39 | return "%(name)s%(labels)s %(value)s" % dict( 40 | name=self.name, 41 | labels=labels_str, 42 | value=self.value 43 | ) 44 | 45 | 46 | class DocRepresentation(BaseRepresentation): 47 | 48 | def __init__(self, name: str, type: str, documentation: str): 49 | self.doc = documentation 50 | self.name = name 51 | self.type = type 52 | 53 | def output(self): 54 | return "# HELP {name} {doc}\n# TYPE {name} {type}".format( 55 | doc=self.doc, 56 | name=self.name, 57 | type=self.type, 58 | ) 59 | 60 | 61 | class WithLabels(object): 62 | """Wrap functions and put 'labels' argument to it.""" 63 | __slot__ = ( 64 | "instance", 65 | "labels", 66 | "wrapped_functions_names", 67 | ) 68 | 69 | def __init__(self, instance, labels: dict, wrapped_functions_names: List[str]): 70 | self.instance = instance 71 | self.labels = labels 72 | self.wrapped_functions_names = wrapped_functions_names 73 | 74 | def __getattr__(self, wrapped_function_name): 75 | if wrapped_function_name not in self.wrapped_functions_names: 76 | raise TypeError("Labels work with functions {} only".format( 77 | self.wrapped_functions_names, 78 | )) 79 | wrapped_function = getattr(self.instance, wrapped_function_name) 80 | return partial(wrapped_function, labels=self.labels) 81 | 82 | 83 | class BaseMetric(object): 84 | """ 85 | Proxy object for real work objects called 'minions'. 86 | Use as global representation on metric. 87 | """ 88 | 89 | minion = None 90 | type = '' 91 | wrapped_functions_names = [] 92 | 93 | def __init__(self, name: str, 94 | documentation: str, labelnames: list=None, 95 | registry: Registry=REGISTRY): 96 | self.documentation = documentation 97 | self.labelnames = labelnames or [] 98 | self.name = name 99 | self.registry = registry 100 | self.registry.add_metric(self) 101 | 102 | def doc_string(self) -> DocRepresentation: 103 | return DocRepresentation( 104 | self.name, 105 | self.type, 106 | self.documentation, 107 | ) 108 | 109 | def get_metric_group_key(self): 110 | return "{}_group".format(self.name) 111 | 112 | def get_metric_key(self, labels, suffix: str=None): 113 | return "{}{}:{}".format( 114 | self.name, 115 | suffix or "", 116 | self.pack_labels(labels).decode('utf-8'), 117 | ) 118 | 119 | def parse_metric_key(self, key) -> (str, dict): 120 | return key.decode('utf-8').split(':', maxsplit=1) 121 | 122 | def pack_labels(self, labels: dict) -> bytes: 123 | return base64.b64encode( 124 | json.dumps(labels, sort_keys=True).encode('utf-8') 125 | ) 126 | 127 | def unpack_labels(self, labels: str) -> dict: 128 | return json.loads(base64.b64decode(labels).decode('utf-8')) 129 | 130 | def _check_labels(self, labels): 131 | if set(labels.keys()) != set(self.labelnames): 132 | raise ValueError("Expect define all labels: {}. Got only: {}".format( 133 | ", ".join(self.labelnames), 134 | ", ".join(labels.keys()) 135 | )) 136 | 137 | def labels(self, *args, **kwargs): 138 | labels = dict(zip(self.labelnames, args)) 139 | labels.update(kwargs) 140 | self._check_labels(labels) 141 | return WithLabels( 142 | instance=self, 143 | labels=labels, 144 | wrapped_functions_names=self.wrapped_functions_names, 145 | ) 146 | 147 | 148 | def silent_wrapper(func): 149 | """Wrap function for process any Exception and write it to log.""" 150 | @wraps(func) 151 | def silent_function(*args, **kwargs): 152 | try: 153 | return func(*args, **kwargs) 154 | except Exception: 155 | logger.exception("Error while send metric to Redis. Function %s", func) 156 | 157 | return silent_function 158 | 159 | 160 | def async_silent_wrapper(func): 161 | """Wrap function for process any Exception and write it to log.""" 162 | @wraps(func) 163 | async def silent_function(*args, **kwargs): 164 | try: 165 | return await func(*args, **kwargs) 166 | except Exception: 167 | logger.exception("Error while send metric to Redis. Function %s", func) 168 | 169 | return silent_function -------------------------------------------------------------------------------- /prometheus_redis_client/helpers.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Callable 3 | from functools import wraps 4 | 5 | 6 | def timeit(metric_callback: Callable, **labels): 7 | def wrapper(func): 8 | @wraps(func) 9 | def func_wrapper(*args, **kwargs): 10 | start = time.time() 11 | result = func(*args, **kwargs) 12 | metric_callback(time.time() - start, labels=labels) 13 | return result 14 | return func_wrapper 15 | return wrapper 16 | -------------------------------------------------------------------------------- /prometheus_redis_client/metrics.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import json 3 | import collections 4 | import threading 5 | from functools import partial 6 | 7 | from prometheus_redis_client.base_metric import BaseMetric, MetricRepresentation, silent_wrapper 8 | from prometheus_redis_client.helpers import timeit 9 | from prometheus_redis_client.registry import Registry, REGISTRY 10 | 11 | DEFAULT_GAUGE_INDEX_KEY = 'GLOBAL_GAUGE_INDEX' 12 | 13 | 14 | class Metric(BaseMetric): 15 | 16 | def collect(self) -> list: 17 | redis = self.registry.redis 18 | group_key = self.get_metric_group_key() 19 | members = redis.smembers(group_key) 20 | 21 | result = [] 22 | for metric_key in members: 23 | name, packed_labels = self.parse_metric_key(metric_key) 24 | labels = self.unpack_labels(packed_labels) 25 | value = redis.get(metric_key) 26 | if value is None: 27 | redis.srem(group_key, metric_key) 28 | continue 29 | result.append(MetricRepresentation( 30 | name=name, 31 | labels=labels, 32 | value=value.decode('utf-8'), 33 | )) 34 | return result 35 | 36 | def cleanup(self): 37 | pass 38 | 39 | 40 | class CommonGauge(Metric): 41 | """Just simple store some value in one key from all processes.""" 42 | 43 | type = 'gauge' 44 | wrapped_functions_names = ['set', 'inc', 'dec'] 45 | 46 | def __init__(self, name: str, 47 | documentation: str, labelnames: list = None, 48 | registry: Registry=REGISTRY, expire: float = None): 49 | """ 50 | Construct CommonGauge metric. 51 | :param name: name of metric 52 | :param documentation: metric description 53 | :param labelnames: list of metric labels 54 | :param registry: the Registry object collect Metric for representation 55 | :param expire: equivalent Redis `expire`; after that timeout Redis delete key. It useful when 56 | you want know if metric does not set a long time. 57 | """ 58 | super().__init__(name, documentation, labelnames, registry) 59 | self._expire = expire 60 | 61 | def set(self, value, labels=None, expire: float = None): 62 | labels = labels or {} 63 | self._check_labels(labels) 64 | if value is None: 65 | raise ValueError('value can not be None') 66 | self._set(value, labels, expire=expire or self._expire) 67 | 68 | @silent_wrapper 69 | def _set(self, value, labels, expire: float = None): 70 | group_key = self.get_metric_group_key() 71 | metric_key = self.get_metric_key(labels) 72 | 73 | pipeline = self.registry.redis.pipeline() 74 | pipeline.sadd(group_key, metric_key) 75 | pipeline.set(metric_key, value, ex=expire) 76 | return pipeline.execute() 77 | 78 | def inc(self, value: float = 1, labels=None, expire: float = None): 79 | labels = labels or {} 80 | self._check_labels(labels) 81 | return self._inc(value, labels, expire=expire or self._expire) 82 | 83 | def dec(self, value: float = 1, labels=None, expire: float = None): 84 | labels = labels or {} 85 | self._check_labels(labels) 86 | return self._inc(-value, labels, expire=expire or self._expire) 87 | 88 | @silent_wrapper 89 | def _inc(self, value: float, labels: dict, expire: float = None): 90 | group_key = self.get_metric_group_key() 91 | metric_key = self.get_metric_key(labels) 92 | pipeline = self.registry.redis.pipeline() 93 | pipeline.sadd(group_key, metric_key) 94 | pipeline.incrbyfloat(metric_key, float(value)) 95 | if expire: 96 | pipeline.expire(metric_key, expire) 97 | return pipeline.execute()[1] 98 | 99 | 100 | class Counter(Metric): 101 | type = 'counter' 102 | wrapped_functions_names = ['inc', 'set'] 103 | 104 | def inc(self, value: int = 1, labels=None): 105 | """ 106 | Calculate metric with labels redis key. 107 | Add this key to set of key for this metric. 108 | """ 109 | labels = labels or {} 110 | self._check_labels(labels) 111 | 112 | if not isinstance(value, int): 113 | raise ValueError("Value should be int, got {}".format( 114 | type(value) 115 | )) 116 | return self._inc(value, labels) 117 | 118 | @silent_wrapper 119 | def _inc(self, value: int, labels: dict): 120 | group_key = self.get_metric_group_key() 121 | metric_key = self.get_metric_key(labels) 122 | 123 | pipeline = self.registry.redis.pipeline() 124 | pipeline.sadd(group_key, metric_key) 125 | pipeline.incrby(metric_key, int(value)) 126 | return pipeline.execute()[1] 127 | 128 | def set(self, value: int = 1, labels=None): 129 | """ 130 | Calculate metric with labels redis key. 131 | Set this key to set of key for this metric. 132 | """ 133 | labels = labels or {} 134 | self._check_labels(labels) 135 | 136 | if not isinstance(value, int): 137 | raise ValueError("Value should be int, got {}".format( 138 | type(value) 139 | )) 140 | return self._set(value, labels) 141 | 142 | @silent_wrapper 143 | def _set(self, value: int, labels: dict): 144 | group_key = self.get_metric_group_key() 145 | metric_key = self.get_metric_key(labels) 146 | 147 | pipeline = self.registry.redis.pipeline() 148 | pipeline.sadd(group_key, metric_key) 149 | pipeline.set(metric_key, int(value)) 150 | return pipeline.execute()[1] 151 | 152 | 153 | class Summary(Metric): 154 | type = 'summary' 155 | wrapped_functions_names = ['observe', ] 156 | 157 | def __init__(self, *args, **kwargs): 158 | super().__init__(*args, **kwargs) 159 | self.timeit = partial(timeit, metric_callback=self.observe) 160 | 161 | def observe(self, value, labels=None): 162 | labels = labels or {} 163 | self._check_labels(labels) 164 | print('DEBUG:', value, labels) 165 | return self._observer(value, labels) 166 | 167 | @silent_wrapper 168 | def _observer(self, value, labels: dict): 169 | group_key = self.get_metric_group_key() 170 | sum_metric_key = self.get_metric_key(labels, "_sum") 171 | count_metric_key = self.get_metric_key(labels, "_count") 172 | 173 | pipeline = self.registry.redis.pipeline() 174 | pipeline.sadd(group_key, count_metric_key, sum_metric_key) 175 | pipeline.incrbyfloat(sum_metric_key, float(value)) 176 | pipeline.incr(count_metric_key) 177 | return pipeline.execute()[1] 178 | 179 | 180 | class Gauge(Metric): 181 | type = 'gauge' 182 | wrapped_functions_names = ['inc', 'set', ] 183 | 184 | default_expire = 60 185 | 186 | def __init__(self, *args, 187 | expire=default_expire, 188 | refresh_enable=True, 189 | gauge_index_key: str = DEFAULT_GAUGE_INDEX_KEY, 190 | **kwargs): 191 | super().__init__(*args, **kwargs) 192 | self.gauge_index_key = gauge_index_key 193 | self.refresh_enable = refresh_enable 194 | self._refresher_added = False 195 | self.lock = threading.Lock() 196 | self.gauge_values = collections.defaultdict(lambda: 0.0) 197 | self.expire = expire 198 | self.index = None 199 | 200 | def add_refresher(self): 201 | if self.refresh_enable and not self._refresher_added: 202 | self.registry.refresher.add_refresh_function( 203 | self.refresh_values, 204 | ) 205 | self._refresher_added = True 206 | 207 | def _set_internal(self, key: str, value: float): 208 | self.gauge_values[key] = value 209 | 210 | def _inc_internal(self, key: str, value: float): 211 | self.gauge_values[key] += value 212 | 213 | def inc(self, value: float, labels: dict = None): 214 | labels = labels or {} 215 | self._check_labels(labels) 216 | return self._inc(value, labels) 217 | 218 | def dec(self, value: float, labels: dict = None): 219 | labels = labels or {} 220 | self._check_labels(labels) 221 | return self._inc(-value, labels) 222 | 223 | @silent_wrapper 224 | def _inc(self, value: float, labels: dict): 225 | with self.lock: 226 | group_key = self.get_metric_group_key() 227 | labels['gauge_index'] = self.get_gauge_index() 228 | metric_key = self.get_metric_key(labels) 229 | 230 | pipeline = self.registry.redis.pipeline() 231 | pipeline.sadd(group_key, metric_key) 232 | pipeline.incrbyfloat(metric_key, float(value)) 233 | pipeline.expire(metric_key, self.expire) 234 | self._inc_internal(metric_key, float(value)) 235 | result = pipeline.execute() 236 | 237 | self.add_refresher() 238 | return result 239 | 240 | def set(self, value: float, labels:dict = None): 241 | labels = labels or {} 242 | self._check_labels(labels) 243 | return self._set(value, labels) 244 | 245 | @silent_wrapper 246 | def _set(self, value: float, labels: dict): 247 | with self.lock: 248 | group_key = self.get_metric_group_key() 249 | labels['gauge_index'] = self.get_gauge_index() 250 | metric_key = self.get_metric_key(labels) 251 | 252 | pipeline = self.registry.redis.pipeline() 253 | pipeline.sadd(group_key, metric_key) 254 | pipeline.set( 255 | metric_key, 256 | float(value), 257 | ex=self.expire, 258 | ) 259 | self._set_internal(metric_key, float(value)) 260 | result = pipeline.execute() 261 | self.add_refresher() 262 | return result 263 | 264 | def get_gauge_index(self): 265 | if self.index is None: 266 | self.index = self.make_gauge_index() 267 | return self.index 268 | 269 | def make_gauge_index(self): 270 | index = self.registry.redis.incr( 271 | self.gauge_index_key, 272 | ) 273 | self.registry.refresher.add_refresh_function( 274 | self.refresh_values, 275 | ) 276 | return index 277 | 278 | def refresh_values(self): 279 | with self.lock: 280 | for key, value in self.gauge_values.items(): 281 | self.registry.redis.set( 282 | key, value, ex=self.expire, 283 | ) 284 | 285 | def cleanup(self): 286 | with self.lock: 287 | group_key = self.get_metric_group_key() 288 | keys = list(self.gauge_values.keys()) 289 | if len(keys) == 0: 290 | return 291 | pipeline = self.registry.redis.pipeline() 292 | pipeline.srem(group_key, *keys) 293 | pipeline.delete(*keys) 294 | pipeline.execute() 295 | 296 | 297 | class Histogram(Metric): 298 | type = 'histogram' 299 | wrapped_functions_names = ['observe', ] 300 | 301 | def __init__(self, *args, buckets: list, **kwargs): 302 | super().__init__(*args, **kwargs) 303 | self.buckets = sorted(buckets, reverse=True) 304 | self.timeit = partial(timeit, metric_callback=self.observe) 305 | 306 | def observe(self, value, labels=None): 307 | labels = labels or {} 308 | self._check_labels(labels) 309 | return self._a_observe(value, labels) 310 | 311 | @silent_wrapper 312 | def _a_observe(self, value: float, labels): 313 | group_key = self.get_metric_group_key() 314 | sum_key = self.get_metric_key(labels, '_sum') 315 | counter_key = self.get_metric_key(labels, '_count') 316 | pipeleine = self.registry.redis.pipeline() 317 | for bucket in self.buckets: 318 | if value > bucket: 319 | break 320 | labels['le'] = bucket 321 | bucket_key = self.get_metric_key(labels, '_bucket') 322 | pipeleine.sadd(group_key, bucket_key) 323 | pipeleine.incr(bucket_key) 324 | pipeleine.sadd(group_key, sum_key, counter_key) 325 | pipeleine.incr(counter_key) 326 | pipeleine.incrbyfloat(sum_key, float(value)) 327 | return pipeleine.execute() 328 | 329 | def _get_missing_metric_values(self, redis_metric_values): 330 | missing_metrics_values = set( 331 | json.dumps({"le": b}) for b in self.buckets 332 | ) 333 | groups = set("{}") 334 | 335 | # If flag is raised then we should add 336 | # *_sum and *_count values for empty labels. 337 | sc_flag = True 338 | for mv in redis_metric_values: 339 | key = json.dumps(mv.labels, sort_keys=True) 340 | labels = copy.copy(mv.labels) 341 | if 'le' in labels: 342 | del labels['le'] 343 | group = json.dumps(labels, sort_keys=True) 344 | if group == "{}": 345 | sc_flag = False 346 | if group not in groups: 347 | for b in self.buckets: 348 | labels['le'] = b 349 | missing_metrics_values.add( 350 | json.dumps(labels, sort_keys=True) 351 | ) 352 | groups.add(group) 353 | if key in missing_metrics_values: 354 | missing_metrics_values.remove(key) 355 | return missing_metrics_values, sc_flag 356 | 357 | def collect(self) -> list: 358 | redis_metrics = super().collect() 359 | missing_metrics_values, sc_flag = self._get_missing_metric_values( 360 | redis_metrics, 361 | ) 362 | 363 | missing_values = [ 364 | MetricRepresentation( 365 | self.name + "_bucket", 366 | labels=json.loads(ls), 367 | value=0, 368 | ) for ls in missing_metrics_values 369 | ] 370 | 371 | if sc_flag: 372 | missing_values.append( 373 | MetricRepresentation( 374 | self.name + "_sum", 375 | labels={}, 376 | value=0, 377 | ), 378 | ) 379 | missing_values.append( 380 | MetricRepresentation( 381 | self.name + "_count", 382 | labels={}, 383 | value=0, 384 | ), 385 | ) 386 | 387 | return redis_metrics + missing_values 388 | -------------------------------------------------------------------------------- /prometheus_redis_client/registry.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | 4 | from redis import StrictRedis 5 | 6 | 7 | class Refresher(object): 8 | 9 | default_refresh_period = 30 10 | 11 | def __init__(self, refresh_period: float = default_refresh_period, timeout_granule=1): 12 | self._refresh_functions_lock = threading.Lock() 13 | self._start_thread_lock = threading.Lock() 14 | self.refresh_period = refresh_period 15 | self.timeout_granule = timeout_granule 16 | self._clean() 17 | 18 | def _clean(self): 19 | self._refresh_functions = [] 20 | self._refresh_enable = False 21 | self._refresh_cycle_thread = threading.Thread(target=self.refresh_cycle) 22 | self._should_be_close = False 23 | 24 | def add_refresh_function(self, func: callable): 25 | with self._refresh_functions_lock: 26 | self._refresh_functions.append(func) 27 | self.start_if_not() 28 | 29 | def start_if_not(self): 30 | with self._start_thread_lock: 31 | if self._refresh_enable: 32 | return 33 | self._refresh_cycle_thread.start() 34 | self._refresh_enable = True 35 | 36 | def cleanup_and_stop(self): 37 | self._should_be_close = True 38 | if self._refresh_enable: 39 | self._refresh_cycle_thread.join() 40 | self._clean() 41 | 42 | def refresh_cycle(self): 43 | """Check `close` flag every `timeout_granule` and refresh after `refresh_period`.""" 44 | current_time_passed = 0 45 | while True: 46 | current_time_passed += self.timeout_granule 47 | if self._should_be_close: 48 | return 49 | if current_time_passed >= self.refresh_period: 50 | current_time_passed = 0 51 | with self._refresh_functions_lock: 52 | 53 | for refresh_func in self._refresh_functions: 54 | refresh_func() 55 | time.sleep(self.timeout_granule) 56 | 57 | 58 | class Registry(object): 59 | 60 | def __init__(self, redis: StrictRedis = None, refresher: Refresher = None): 61 | self._metrics = [] 62 | self.redis = None 63 | self.refresher = refresher or Refresher() 64 | self.set_redis(redis) 65 | 66 | def output(self) -> str: 67 | all_metric = [] 68 | for metric in self._metrics: 69 | all_metric.append(metric.doc_string()) 70 | ms = metric.collect() 71 | all_metric += sorted([ 72 | p for p in ms 73 | ], key=lambda x: x.output()) 74 | return "\n".join(( 75 | m.output() for m in all_metric 76 | )) 77 | 78 | def add_metric(self, *metrics): 79 | already_added = set([ 80 | m.name for m in self._metrics 81 | ]) 82 | new_metrics = set([ 83 | m.name for m in metrics 84 | ]) 85 | doubles = already_added.intersection(new_metrics) 86 | if doubles: 87 | raise ValueError("Metrics {} already added".format( 88 | ", ".join(doubles), 89 | )) 90 | 91 | for m in metrics: 92 | self._metrics.append(m) 93 | 94 | def set_redis(self, redis): 95 | self.redis = redis 96 | 97 | def set_refresher(self, refresher: Refresher): 98 | self.refresher = refresher 99 | 100 | def cleanup_and_stop(self): 101 | if self.refresher: 102 | self.refresher.cleanup_and_stop() 103 | for metric in self._metrics: 104 | metric.cleanup() 105 | self._metrics = [] 106 | 107 | 108 | REGISTRY = Registry() 109 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==4.3.1 2 | redis==4.0.2 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='prometheus_redis_client', 5 | packages=['prometheus_redis_client'], 6 | version='0.6.0', 7 | description='Python prometheus multiprocessing client which used redis as metric storage.', 8 | author='Belousov Alex', 9 | author_email='belousov.aka.alfa@gmail.com', 10 | url='https://github.com/belousovalex/prometheus_redis_client', 11 | install_requires=['redis>=3.2.1,<5.0.0', ], 12 | license='Apache 2', 13 | ) 14 | -------------------------------------------------------------------------------- /test-docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | redis: 4 | image: redis:latest 5 | tests: 6 | build: 7 | context: . 8 | dockerfile: Dockerfile.tests 9 | depends_on: 10 | - redis -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/belousovalex/prometheus_redis_client/90852fb0eaf3aee1937a74cad6181c304dc6999a/tests/__init__.py -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | import redis 4 | import prometheus_redis_client as prom 5 | 6 | 7 | @contextmanager 8 | def MetricEnvironment(): 9 | redis_client = redis.from_url("redis://redis:6379") 10 | redis_client.flushdb() 11 | refresher = prom.Refresher(refresh_period=2) 12 | prom.REGISTRY.set_redis(redis_client) 13 | prom.REGISTRY.set_refresher(refresher) 14 | try: 15 | yield redis_client 16 | finally: 17 | prom.REGISTRY.cleanup_and_stop() 18 | -------------------------------------------------------------------------------- /tests/test_common_gauge.py: -------------------------------------------------------------------------------- 1 | import time 2 | from unittest.mock import patch 3 | import pytest 4 | 5 | from .helpers import MetricEnvironment 6 | import prometheus_redis_client as prom 7 | 8 | 9 | class TestCommonGauge(object): 10 | 11 | def test_none_value_exception(self): 12 | with MetricEnvironment(): 13 | 14 | const = prom.CommonGauge( 15 | name="test_const1", 16 | documentation="Const metric documentation", 17 | ) 18 | with pytest.raises(ValueError, match=r"value can not be None"): 19 | const.set(None) 20 | 21 | def test_interface_without_labels(self): 22 | with MetricEnvironment() as redis: 23 | 24 | const = prom.CommonGauge( 25 | name="test_const1", 26 | documentation="Const metric documentation", 27 | ) 28 | 29 | const.set(12) 30 | group_key = const.get_metric_group_key() 31 | metric_key = const.get_metric_key({}) 32 | 33 | assert redis.smembers(group_key) == {b'test_const1:e30='} 34 | assert int(redis.get(metric_key)) == 12 35 | 36 | const.set(3) 37 | assert float(redis.get(metric_key)) == 3 38 | 39 | const.inc(1.2) 40 | assert float(redis.get(metric_key)) == 4.2 41 | 42 | const.dec(2.1) 43 | assert float(redis.get(metric_key)) == 2.1 44 | 45 | assert (prom.REGISTRY.output()) == ( 46 | "# HELP test_const1 Const metric documentation\n" 47 | "# TYPE test_const1 gauge\n" 48 | "test_const1 2.1" 49 | ) 50 | 51 | def test_interface_with_labels(self): 52 | with MetricEnvironment() as redis: 53 | 54 | const = prom.CommonGauge( 55 | name="test_const2", 56 | documentation="Const documentation", 57 | labelnames=["host", "url"] 58 | ) 59 | 60 | # need 'url' label 61 | with pytest.raises(ValueError, match="Expect define all labels: host, url. Got only: host"): 62 | const.labels(host="123.123.123.123").set(12) 63 | 64 | # need use labels method 65 | with pytest.raises(Exception): 66 | const.set(12) 67 | 68 | labels = dict(host="123.123.123.123", url="/home/") 69 | const.labels(**labels).set(2) 70 | group_key = const.get_metric_group_key() 71 | metric_key = const.get_metric_key(labels) 72 | 73 | assert redis.smembers(group_key) == {b'test_const2:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0='} 74 | assert int(redis.get(metric_key)) == 2 75 | 76 | const.labels(**labels).set(3) 77 | assert int(redis.get(metric_key)) == 3 78 | 79 | const.labels(**labels).inc(1.2) 80 | assert float(redis.get(metric_key)) == 4.2 81 | 82 | const.labels(**labels).dec(2.1) 83 | assert float(redis.get(metric_key)) == 2.1 84 | 85 | assert (prom.REGISTRY.output()) == ( 86 | "# HELP test_const2 Const documentation\n" 87 | "# TYPE test_const2 gauge\n" 88 | "test_const2{host=\"123.123.123.123\",url=\"/home/\"} 2.1" 89 | ) 90 | 91 | @patch('prometheus_redis_client.base_metric.logger.exception') 92 | def test_expire(self, mock_lgger): 93 | """ 94 | Test expire mode for CommonGauge metrics. 95 | If we set expire param in _init__ then after `expire` seconds Redis delete our metric value. 96 | Other value should be available. 97 | """ 98 | with MetricEnvironment() as redis: 99 | const = prom.CommonGauge( 100 | name="test_const2", 101 | documentation="Const documentation", 102 | labelnames=["host", "url"], 103 | expire=2, 104 | ) 105 | 106 | const.labels(host="123.123.123.123", url="/home/").set(12, expire=3) 107 | const.labels(host="124.124.124.124", url="/home/").set(24) # its should be expire after 2 seconds 108 | 109 | assert (prom.REGISTRY.output()) == ( 110 | "# HELP test_const2 Const documentation\n" 111 | "# TYPE test_const2 gauge\n" 112 | "test_const2{host=\"123.123.123.123\",url=\"/home/\"} 12\n" 113 | "test_const2{host=\"124.124.124.124\",url=\"/home/\"} 24" 114 | ) 115 | 116 | time.sleep(1) 117 | const.labels(host="136.136.136.136", url="/home/").set(36) 118 | 119 | assert (prom.REGISTRY.output()) == ( 120 | "# HELP test_const2 Const documentation\n" 121 | "# TYPE test_const2 gauge\n" 122 | "test_const2{host=\"123.123.123.123\",url=\"/home/\"} 12\n" 123 | "test_const2{host=\"124.124.124.124\",url=\"/home/\"} 24\n" 124 | "test_const2{host=\"136.136.136.136\",url=\"/home/\"} 36" 125 | ) 126 | time.sleep(1.5) 127 | 128 | assert (prom.REGISTRY.output()) == ( 129 | "# HELP test_const2 Const documentation\n" 130 | "# TYPE test_const2 gauge\n" 131 | "test_const2{host=\"123.123.123.123\",url=\"/home/\"} 12\n" 132 | "test_const2{host=\"136.136.136.136\",url=\"/home/\"} 36" 133 | ) 134 | 135 | time.sleep(1) 136 | assert (prom.REGISTRY.output()) == ( 137 | "# HELP test_const2 Const documentation\n" 138 | "# TYPE test_const2 gauge" 139 | ) 140 | 141 | @patch('prometheus_redis_client.base_metric.logger.exception') 142 | def test_silent_mode(self, mock_logger): 143 | """ 144 | If we have some errors while send metric 145 | to redis we should not stop usefull work. 146 | """ 147 | with MetricEnvironment(): 148 | 149 | const = prom.CommonGauge( 150 | name="test_counter2", 151 | documentation="Counter documentation" 152 | ) 153 | 154 | def raised_exception_func(*args, **kwargs): 155 | raise Exception() 156 | 157 | with patch("prometheus_redis_client.REGISTRY.redis.pipeline") as mock: 158 | mock.side_effect = raised_exception_func 159 | 160 | const.set(12) 161 | assert mock_logger.called 162 | -------------------------------------------------------------------------------- /tests/test_counters.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | import pytest 3 | 4 | from .helpers import MetricEnvironment 5 | import prometheus_redis_client as prom 6 | 7 | 8 | class TestCounter(object): 9 | 10 | def test_interface_without_labels(self): 11 | with MetricEnvironment() as redis: 12 | 13 | counter = prom.Counter( 14 | name="test_counter1", 15 | documentation="Counter documentation" 16 | ) 17 | 18 | counter.inc() 19 | group_key = counter.get_metric_group_key() 20 | metric_key = counter.get_metric_key({}) 21 | 22 | assert redis.smembers(group_key) == {b'test_counter1:e30='} 23 | assert int(redis.get(metric_key)) == 1 24 | 25 | counter.inc(3) 26 | assert float(redis.get(metric_key)) == 4 27 | 28 | assert (prom.REGISTRY.output()) == ( 29 | "# HELP test_counter1 Counter documentation\n" 30 | "# TYPE test_counter1 counter\n" 31 | "test_counter1 4" 32 | ) 33 | 34 | def test_interface_with_labels(self): 35 | with MetricEnvironment() as redis: 36 | 37 | counter = prom.Counter( 38 | name="test_counter2", 39 | documentation="Counter documentation", 40 | labelnames=["host", "url"] 41 | ) 42 | 43 | # need 'url' label 44 | with pytest.raises(ValueError): 45 | counter.labels(host="123.123.123.123").inc() 46 | 47 | # need use labels method 48 | with pytest.raises(Exception): 49 | counter.inc() 50 | 51 | labels = dict(host="123.123.123.123", url="/home/") 52 | counter.labels(**labels).inc(2) 53 | group_key = counter.get_metric_group_key() 54 | metric_key = counter.get_metric_key(labels) 55 | 56 | assert redis.smembers(group_key) == {b'test_counter2:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0='} 57 | assert int(redis.get(metric_key)) == 2 58 | 59 | assert counter.labels(**labels).inc(3) == 5 60 | assert int(redis.get(metric_key)) == 5 61 | 62 | @patch('prometheus_redis_client.base_metric.logger.exception') 63 | def test_silent_mode(self, mock_logger): 64 | """ 65 | If we have some errors while send metric 66 | to redis we should not stop usefull work. 67 | """ 68 | with MetricEnvironment(): 69 | 70 | counter = prom.Counter( 71 | name="test_counter2", 72 | documentation="Counter documentation" 73 | ) 74 | 75 | def raised_exception_func(*args, **kwargs): 76 | raise Exception() 77 | 78 | with patch("prometheus_redis_client.REGISTRY.redis.pipeline") as mock: 79 | mock.side_effect = raised_exception_func 80 | 81 | counter.inc() 82 | assert mock_logger.called 83 | 84 | 85 | def test_add_interface_without_labels(self): 86 | with MetricEnvironment() as redis: 87 | 88 | counter = prom.Counter( 89 | name="test_counter1", 90 | documentation="Counter documentation" 91 | ) 92 | 93 | counter.set(1) 94 | group_key = counter.get_metric_group_key() 95 | metric_key = counter.get_metric_key({}) 96 | 97 | assert redis.smembers(group_key) == {b'test_counter1:e30='} 98 | assert int(redis.get(metric_key)) == 1 99 | 100 | counter.set(30) 101 | assert float(redis.get(metric_key)) == 30 102 | 103 | counter.set(10) 104 | assert float(redis.get(metric_key)) == 10 105 | 106 | assert (prom.REGISTRY.output()) == ( 107 | "# HELP test_counter1 Counter documentation\n" 108 | "# TYPE test_counter1 counter\n" 109 | "test_counter1 10" 110 | ) -------------------------------------------------------------------------------- /tests/test_gauge.py: -------------------------------------------------------------------------------- 1 | import time 2 | from unittest.mock import patch 3 | 4 | import pytest 5 | import base64 6 | 7 | from .helpers import MetricEnvironment 8 | import prometheus_redis_client as prom 9 | 10 | 11 | class TestGauge(object): 12 | 13 | def test_interface_without_labels(self): 14 | with MetricEnvironment() as redis: 15 | gauge = prom.Gauge( 16 | "test_gauge", 17 | "Gauge Documentation", 18 | expire=4, 19 | ) 20 | 21 | gauge.set(12.3) 22 | gauge_index = int(redis.get(prom.DEFAULT_GAUGE_INDEX_KEY)) 23 | 24 | group_key = gauge.get_metric_group_key() 25 | metric_key = "test_gauge:{}".format( 26 | base64.b64encode(('{"gauge_index": %s}' % gauge_index).encode('utf-8')).decode('utf-8') 27 | ).encode('utf-8') 28 | 29 | assert sorted(redis.smembers(group_key)) == sorted([ 30 | metric_key, 31 | ]) 32 | assert float(redis.get(metric_key)) == 12.3 33 | 34 | gauge.set(12.9) 35 | assert sorted(redis.smembers(group_key)) == sorted([ 36 | metric_key, 37 | ]) 38 | assert float(redis.get(metric_key)) == 12.9 39 | 40 | gauge.dec(1.7) 41 | assert sorted(redis.smembers(group_key)) == sorted([ 42 | metric_key 43 | ]) 44 | assert float(redis.get(metric_key)) == 11.2 45 | 46 | assert prom.REGISTRY.output() == ( 47 | "# HELP test_gauge Gauge Documentation\n" 48 | "# TYPE test_gauge gauge\n" 49 | "test_gauge{gauge_index=\"%s\"} 11.2" 50 | ) % gauge_index 51 | 52 | def test_interface_with_labels(self): 53 | with MetricEnvironment() as redis: 54 | gauge = prom.Gauge( 55 | "test_gauge", 56 | "Gauge Documentation", 57 | ['name'], 58 | expire=4, 59 | ) 60 | 61 | gauge.labels(name='test').set(12.3) 62 | 63 | gauge_index = int(redis.get(prom.DEFAULT_GAUGE_INDEX_KEY)) 64 | group_key = gauge.get_metric_group_key() 65 | metric_key = "test_gauge:{}".format( 66 | base64.b64encode(('{"gauge_index": %s, "name": "test"}' % gauge_index).encode('utf-8')).decode('utf-8') 67 | ).encode('utf-8') 68 | assert sorted(redis.smembers(group_key)) == sorted([ 69 | metric_key, 70 | ]) 71 | assert float(redis.get(metric_key)) == 12.3 72 | 73 | gauge.labels(name='test').inc(1.7) 74 | assert sorted(redis.smembers(group_key)) == sorted([ 75 | metric_key, 76 | ]) 77 | assert float(redis.get(metric_key)) == 14.0 78 | 79 | assert prom.REGISTRY.output() == ( 80 | "# HELP test_gauge Gauge Documentation\n" 81 | "# TYPE test_gauge gauge\n" 82 | "test_gauge{gauge_index=\"%s\",name=\"test\"} 14" 83 | ) % gauge_index 84 | 85 | def test_auto_clean(self): 86 | with MetricEnvironment() as redis: 87 | gauge = prom.Gauge( 88 | "test_gauge", 89 | "Gauge Documentation", 90 | expire=4, 91 | ) 92 | gauge.set(12.3) 93 | 94 | group_key = gauge.get_metric_group_key() 95 | gauge_index = int(redis.get(prom.DEFAULT_GAUGE_INDEX_KEY)) 96 | metric_key = "test_gauge:{}".format( 97 | base64.b64encode(('{"gauge_index": %s}' % gauge_index).encode('utf-8')).decode('utf-8') 98 | ).encode('utf-8') 99 | assert float(redis.get(metric_key)) == 12.3 100 | 101 | # force stop refresh metrics 102 | prom.REGISTRY.refresher.cleanup_and_stop() 103 | 104 | # after expire timeout metric should be remove 105 | time.sleep(5) 106 | assert redis.get(metric_key) is None 107 | 108 | assert prom.REGISTRY.output() == ( 109 | "# HELP test_gauge Gauge Documentation\n" 110 | "# TYPE test_gauge gauge" 111 | ) 112 | # ... and remove metric from group 113 | assert redis.smembers(group_key) == set() 114 | 115 | def test_refresh(self): 116 | with MetricEnvironment() as redis: 117 | gauge = prom.Gauge( 118 | "test_gauge", 119 | "Gauge Documentation", 120 | expire=4, 121 | ) 122 | 123 | gauge.set(12.3) 124 | 125 | gauge_index = int(redis.get(prom.DEFAULT_GAUGE_INDEX_KEY)) 126 | metric_key = "test_gauge:{}".format( 127 | base64.b64encode(('{"gauge_index": %s}' % gauge_index).encode('utf-8')).decode('utf-8') 128 | ).encode('utf-8') 129 | assert float(redis.get(metric_key)) == 12.3 130 | 131 | time.sleep(6) 132 | assert float(redis.get(metric_key)) == 12.3 133 | 134 | assert (prom.REGISTRY.output()) == ( 135 | "# HELP test_gauge Gauge Documentation\n" 136 | "# TYPE test_gauge gauge\n" 137 | "test_gauge{gauge_index=\"%s\"} 12.3" 138 | ) % gauge_index 139 | -------------------------------------------------------------------------------- /tests/test_histogram.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | import pytest 4 | 5 | from .helpers import MetricEnvironment 6 | import prometheus_redis_client as prom 7 | 8 | 9 | class TestHistogram(object): 10 | 11 | def test_interface_without_labels(self): 12 | with MetricEnvironment() as redis: 13 | 14 | histogram = prom.Histogram( 15 | name="test_histogram", 16 | documentation="Histogram documentation", 17 | buckets=[1, 20, 25.5] 18 | ) 19 | 20 | histogram.observe(25.4) 21 | group_key = histogram.get_metric_group_key() 22 | 23 | assert sorted(redis.smembers(group_key)) == [ 24 | b'test_histogram_bucket:eyJsZSI6IDI1LjV9', 25 | b'test_histogram_count:e30=', 26 | b'test_histogram_sum:e30=', 27 | ] 28 | assert float(redis.get('test_histogram_sum:e30=')) == 25.4 29 | assert float(redis.get('test_histogram_bucket:eyJsZSI6IDI1LjV9')) == 1 30 | assert float(redis.get('test_histogram_count:e30=')) == 1 31 | 32 | histogram.observe(3) 33 | 34 | assert sorted(redis.smembers(group_key)) == [ 35 | b'test_histogram_bucket:eyJsZSI6IDI1LjV9', 36 | b'test_histogram_bucket:eyJsZSI6IDIwfQ==', 37 | b'test_histogram_count:e30=', 38 | b'test_histogram_sum:e30=' 39 | ] 40 | assert float(redis.get('test_histogram_sum:e30=')) == 28.4 41 | assert float(redis.get('test_histogram_count:e30=')) == 2 42 | assert float(redis.get('test_histogram_bucket:eyJsZSI6IDIwfQ==')) == 1 43 | assert float(redis.get('test_histogram_bucket:eyJsZSI6IDI1LjV9')) == 2 44 | 45 | assert prom.REGISTRY.output() == ( 46 | '# HELP test_histogram Histogram documentation\n' 47 | '# TYPE test_histogram histogram\n' 48 | 'test_histogram_bucket{le="1"} 0\n' 49 | 'test_histogram_bucket{le="20"} 1\n' 50 | 'test_histogram_bucket{le="25.5"} 2\n' 51 | 'test_histogram_count 2\n' 52 | 'test_histogram_sum 28.4' 53 | ) 54 | 55 | def test_interface_with_labels(self): 56 | with MetricEnvironment() as redis: 57 | 58 | histogram = prom.Histogram( 59 | name="test_histogram", 60 | documentation="Histogram documentation", 61 | labelnames=["host", "url"], 62 | buckets=[0, 1, 2.001, 3], 63 | ) 64 | 65 | # need 'url' label 66 | with pytest.raises(ValueError): 67 | histogram.labels(host="123.123.123.123").observe(12) 68 | 69 | # need use labels method 70 | with pytest.raises(Exception): 71 | histogram.observe(23) 72 | 73 | labels = dict(host="123.123.123.123", url="/home/") 74 | histogram.labels("123.123.123.123", "/home/").observe(2.1) 75 | group_key = histogram.get_metric_group_key() 76 | 77 | counter_key = b'test_histogram_count:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0=' 78 | sum_key = b'test_histogram_sum:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0=' 79 | key_bucket_3 = b'test_histogram_bucket:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJsZSI6IDMsICJ1cmwiOiAiL2hvbWUvIn0=' 80 | 81 | key_bucket_1 = b'test_histogram_bucket:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJsZSI6IDEsICJ1cmwiOiAiL2hvbWUvIn0=' 82 | key_bucket_2_001 = b'test_histogram_bucket:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJsZSI6IDIuMDAxLCAidXJsIjogIi9ob21lLyJ9' 83 | 84 | assert sorted(redis.smembers(group_key)) == [ 85 | key_bucket_3, counter_key, sum_key, 86 | ] 87 | assert float(redis.get(key_bucket_3)) == 1 88 | 89 | histogram.labels(**labels).observe(0.2) 90 | 91 | assert sorted(redis.smembers(group_key)) == sorted([ 92 | key_bucket_1, 93 | key_bucket_2_001, 94 | key_bucket_3, 95 | counter_key, sum_key 96 | ]) 97 | assert float(redis.get(key_bucket_3)) == 2 98 | assert float(redis.get(key_bucket_1)) == 1 99 | assert float(redis.get(sum_key)) == 2.3 100 | assert float(redis.get(counter_key)) == 2 101 | 102 | assert prom.REGISTRY.output() == ( 103 | '# HELP test_histogram Histogram documentation\n' 104 | '# TYPE test_histogram histogram\n' 105 | 'test_histogram_bucket{host="123.123.123.123",le="0",url="/home/"} 0\n' 106 | 'test_histogram_bucket{host="123.123.123.123",le="1",url="/home/"} 1\n' 107 | 'test_histogram_bucket{host="123.123.123.123",le="2.001",url="/home/"} 1\n' 108 | 'test_histogram_bucket{host="123.123.123.123",le="3",url="/home/"} 2\n' 109 | 'test_histogram_bucket{le="0"} 0\n' 110 | 'test_histogram_bucket{le="1"} 0\n' 111 | 'test_histogram_bucket{le="2.001"} 0\n' 112 | 'test_histogram_bucket{le="3"} 0\n' 113 | 'test_histogram_count 0\n' 114 | 'test_histogram_count{host="123.123.123.123",url="/home/"} 2\n' 115 | 'test_histogram_sum 0\n' 116 | 'test_histogram_sum{host="123.123.123.123",url="/home/"} 2.3' 117 | ) 118 | 119 | @patch('prometheus_redis_client.base_metric.logger.exception') 120 | def test_silent_mode(self, mock_logger): 121 | """ 122 | If we have some errors while send metric 123 | to redis we should not stop usefull work. 124 | """ 125 | with MetricEnvironment(): 126 | 127 | histogram = prom.Histogram( 128 | name="test_histogram", 129 | documentation="Histogram documentation", 130 | buckets=[0, 1, 2.001, 3], 131 | ) 132 | 133 | def raised_exception_func(*args, **kwargs): 134 | raise Exception() 135 | 136 | with patch("prometheus_redis_client.REGISTRY.redis.pipeline") as mock: 137 | mock.side_effect = raised_exception_func 138 | 139 | histogram.observe(1) 140 | assert mock_logger.called 141 | 142 | def test_timeit_wrapper(self): 143 | """Test `timeit` wrapper for Histogram metric.""" 144 | 145 | with MetricEnvironment(): 146 | 147 | histogram = prom.Histogram( 148 | name="test_histogram", 149 | documentation="Histogram documentation", 150 | buckets=[0, 1, 2.001, 3], 151 | ) 152 | 153 | @histogram.timeit() 154 | def simple_func(): 155 | import time 156 | time.sleep(0.01) 157 | return 158 | 159 | simple_func() 160 | 161 | assert prom.REGISTRY.output().startswith( 162 | '# HELP test_histogram Histogram documentation\n' 163 | '# TYPE test_histogram histogram\n' 164 | 'test_histogram_bucket{le="0"} 0\n' 165 | 'test_histogram_bucket{le="1"} 1\n' 166 | 'test_histogram_bucket{le="2.001"} 1\n' 167 | 'test_histogram_bucket{le="3"} 1\n' 168 | 'test_histogram_count 1\n' 169 | 'test_histogram_sum 0.01' 170 | ) 171 | -------------------------------------------------------------------------------- /tests/test_summary.py: -------------------------------------------------------------------------------- 1 | import re 2 | from unittest.mock import patch 3 | 4 | import pytest 5 | 6 | from .helpers import MetricEnvironment 7 | import prometheus_redis_client as prom 8 | 9 | 10 | class TestSummary(object): 11 | 12 | def test_interface_without_labels(self): 13 | with MetricEnvironment() as redis: 14 | 15 | summary = prom.Summary( 16 | name="test_summary", 17 | documentation="Summary documentation", 18 | ) 19 | 20 | summary.observe(1) 21 | group_key = summary.get_metric_group_key() 22 | 23 | assert sorted(redis.smembers(group_key)) == [ 24 | b'test_summary_count:e30=', 25 | b'test_summary_sum:e30=' 26 | ] 27 | assert int(redis.get('test_summary_count:e30=')) == 1 28 | assert float(redis.get('test_summary_sum:e30=')) == 1 29 | 30 | summary.observe(3.5) 31 | assert int(redis.get('test_summary_count:e30=')) == 2 32 | assert float(redis.get('test_summary_sum:e30=')) == 4.5 33 | 34 | assert prom.REGISTRY.output() == ( 35 | "# HELP test_summary Summary documentation\n" 36 | "# TYPE test_summary summary\n" 37 | "test_summary_count 2\n" 38 | "test_summary_sum 4.5" 39 | ) 40 | 41 | def test_interface_with_labels(self): 42 | with MetricEnvironment() as redis: 43 | 44 | summary = prom.Summary( 45 | name="test_summary", 46 | documentation="Summary documentation", 47 | labelnames=["host", "url", ], 48 | ) 49 | 50 | # need 'url' label 51 | with pytest.raises(ValueError, match=r"Expect define all labels: .*?\. Got only: host"): 52 | summary.labels(host="123.123.123.123").observe(3) 53 | 54 | # need use labels method 55 | with pytest.raises(Exception, match=r"Expect define all labels: .*?\. Got only: "): 56 | summary.observe(1) 57 | 58 | labels = dict(host="123.123.123.123", url="/home/") 59 | summary.labels(**labels).observe(2) 60 | group_key = summary.get_metric_group_key() 61 | 62 | assert sorted(redis.smembers(group_key)) == [ 63 | b'test_summary_count:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0=', 64 | b'test_summary_sum:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0=', 65 | ] 66 | metric_sum_key = 'test_summary_sum:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0=' 67 | metric_count_key = 'test_summary_count:eyJob3N0IjogIjEyMy4xMjMuMTIzLjEyMyIsICJ1cmwiOiAiL2hvbWUvIn0=' 68 | assert int(redis.get(metric_count_key)) == 1 69 | assert float(redis.get(metric_sum_key)) == 2 70 | 71 | assert summary.labels(**labels).observe(3.1) == 5.1 72 | assert int(redis.get(metric_count_key)) == 2 73 | assert float(redis.get(metric_sum_key)) == 5.1 74 | 75 | assert prom.REGISTRY.output() == ( 76 | '# HELP test_summary Summary documentation\n' 77 | '# TYPE test_summary summary\n' 78 | 'test_summary_count{host="123.123.123.123",url="/home/"} 2\n' 79 | 'test_summary_sum{host="123.123.123.123",url="/home/"} 5.1' 80 | ) 81 | 82 | @patch('prometheus_redis_client.base_metric.logger.exception') 83 | def test_silent_mode(self, mock_logger): 84 | """ 85 | If we have some errors while send metric 86 | to redis we should not stop usefull work. 87 | """ 88 | with MetricEnvironment(): 89 | 90 | summary = prom.Summary( 91 | name="test_summary", 92 | documentation="Summary documentation", 93 | ) 94 | 95 | def raised_exception_func(*args, **kwargs): 96 | raise Exception() 97 | 98 | with patch("prometheus_redis_client.REGISTRY.redis.pipeline") as mock: 99 | mock.side_effect = raised_exception_func 100 | 101 | summary.observe(1) 102 | assert mock_logger.called 103 | 104 | def test_timeit_wrapper(self): 105 | """Test `timeit` wrapper for Summary metric.""" 106 | 107 | with MetricEnvironment(): 108 | 109 | summary = prom.Summary( 110 | name="test_summary", 111 | documentation="Summary documentation", 112 | labelnames=['name'], 113 | ) 114 | 115 | @summary.timeit(name="Hi!") 116 | def simple_func(): 117 | import time 118 | time.sleep(0.01) 119 | return 120 | 121 | simple_func() 122 | 123 | assert prom.REGISTRY.output().startswith( 124 | '# HELP test_summary Summary documentation\n' 125 | '# TYPE test_summary summary\n' 126 | 'test_summary_count{name="Hi!"} 1\n' 127 | 'test_summary_sum{name="Hi!"} 0.01' 128 | ) 129 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py36,py37,py38}-redis321 4 | {py36,py37,py38}-redis402 5 | 6 | [testenv] 7 | deps = 8 | pytest==4.3.1 9 | redis321: redis==3.2.1 10 | redis402: redis==4.0.2 11 | commands = 12 | pytest -vv 13 | 14 | --------------------------------------------------------------------------------