├── .gitignore
├── .travis.yml
├── AUTHORS
├── CHANGELOG.rst
├── LICENSE
├── MANIFEST.in
├── Makefile
├── README.rst
├── TODO.org
├── ci
    ├── setup
    └── test
├── conftest.py
├── docs
    └── usage.rst
├── examples
    └── test1.py
├── pyprometheus
    ├── __init__.py
    ├── compat.py
    ├── const.py
    ├── contrib
    │   ├── __init__.py
    │   ├── graphite.py
    │   └── uwsgi_features.py
    ├── managers.py
    ├── metrics.py
    ├── registry.py
    ├── scripts
    │   └── __init__.py
    ├── storage.py
    ├── utils
    │   ├── __init__.py
    │   └── exposition.py
    └── values.py
├── setup.cfg
├── setup.py
├── tests
    ├── __init__.py
    ├── test_core.py
    ├── test_metrics.py
    ├── test_registry.py
    ├── test_storage.py
    └── test_uwsgi_collector.py
├── tests_requirements.txt
├── tools
    ├── Dockerfile
    ├── Makefile
    └── compose-config.yml
└── tox.ini


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.pyc
 2 | *.pyo
 3 | *.egg-info
 4 | __pycache__/*
 5 | .cache/*
 6 | .eggs/*
 7 | .tox/*
 8 | dist/*
 9 | build/*
10 | .coverage
11 | \#*\#
12 | *~
13 | *.\#*
14 | .ropeproject/*
15 | .python-version


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | sudo: false
 3 | 
 4 | 
 5 | install:
 6 |   - time ci/setup
 7 | 
 8 | script:
 9 |   - time ci/test
10 | 
11 | 
12 | notifications:
13 |   email: false
14 | 
15 | 
16 | matrix:
17 |   include:
18 |     - python: "2.7"
19 |       env: UWSGI="2.0.14"
20 | 
21 |     # - python: "3.3"
22 |     #   env: UWSGI="2.0.14"
23 | 
24 |     # - python: "3.4"
25 |     #   env: UWSGI="2.0.14"
26 | 
27 |     # - python: "3.5"
28 |     #   env: UWSGI="2.0.14"
29 | 
30 |     # - python: "3.6"
31 |     #   env: UWSGI="2.0.14"
32 | 


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
 1 | pyprometheus is written and maintained by Alexandr Lispython and
 2 | various contributors:
 3 | 
 4 | Development Lead
 5 | ~~~~~~~~~~~~~~~~
 6 | 
 7 | - Alex Lispython
 8 | 
 9 | Patches and Suggestions
10 | ~~~~~~~~~~~~~~~~~~~~~~~
11 | 


--------------------------------------------------------------------------------
/CHANGELOG.rst:
--------------------------------------------------------------------------------
 1 | Version 0.0.9
 2 | -------------
 3 | 
 4 | * [BUGFIX] Fixed metric samples caching
 5 | 
 6 | 
 7 | Version 0.0.8
 8 | -------------
 9 | 
10 | * [BUGFIX] Fixed escaping for export formats
11 | 
12 | Version 0.0.7
13 | -------------
14 | 
15 | * [BUGFIX] Fixed `UWSGIStorage` memory corruptions
16 | 
17 | 
18 | Version 0.0.6
19 | -------------
20 | 
21 | * [BUGFIX] Fixed `UWSGIFlushStorage` init
22 | 
23 | Version 0.0.5
24 | -------------
25 | 
26 | * [FEATURE] Added `UWSGIFlushStorage`
27 | * [FEATURE] Added `UWSGIStorage` metrics
28 | 
29 | Version 0.0.4
30 | -------------
31 | 
32 | * [BUGFIX] Fixed `UWSGICollector.collect`` to return unique metrics.
33 | * [BUGFIX] Fixed registry_to_text format.
34 | 
35 | Version 0.0.2
36 | -------------
37 | 
38 | * Initial version
39 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2017 by Alexandr Lispython and contributors.
 2 | See AUTHORS for more details.
 3 | 
 4 | Some rights reserved.
 5 | 
 6 | Redistribution and use in source and binary forms of the software as well
 7 | as documentation, with or without modification, are permitted provided
 8 | that the following conditions are met:
 9 | 
10 | * Redistributions of source code must retain the above copyright
11 |   notice, this list of conditions and the following disclaimer.
12 | 
13 | * Redistributions in binary form must reproduce the above
14 |   copyright notice, this list of conditions and the following
15 |   disclaimer in the documentation and/or other materials provided
16 |   with the distribution.
17 | 
18 | * The names of the contributors may not be used to endorse or
19 |   promote products derived from this software without specific
20 |   prior written permission.
21 | 
22 | THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
23 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
24 | NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
26 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
27 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
28 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
29 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
30 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
31 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32 | SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
33 | DAMAGE.


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include setup.py README.rst MANIFEST.in LICENSE *.txt
2 | recursive-include pyprometheus/ *
3 | graft tests
4 | global-exclude *~
5 | global-exclude #*#


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | ifdef CMD
 2 | 	DOCKER_CMD=$(CMD)
 3 | else
 4 | 	DOCKER_CMD=/bin/bash
 5 | endif
 6 | 
 7 | DOCKER_RUN_COMMAND=docker-compose -f tools/compose-config.yml run dev_pyprometheus
 8 | 
 9 | 
10 | default: help
11 | 
12 | include tools/Makefile
13 | 
14 | 
15 | version := $(shell sh -c "egrep -oe '__version__\s+=\s+(.*)' ./pyprometheus/__init__.py | sed 's/ //g' | sed \"s/'//g\" | sed 's/__version__=//g'")
16 | 
17 | #version := $(shell sh -c "$(DOCKER_RUN_COMMAND) 'python setup.py --version'")
18 | clean-pyc:
19 | 	find . -name '*.pyc' -exec rm -f {} +
20 | 	find . -name '*.pyo' -exec rm -f {} +
21 | 	find . -name '*~' -exec rm -f {} +
22 | 	find . -name "__pycache__" -exec rm -rf {} +
23 | 
24 | clean-dist:
25 | 	rm -rf dist/*
26 | 	rm -rf build/*
27 | 
28 | help:
29 | 	@echo "Available commands:"
30 | 	@sed -n '/^[a-zA-Z0-9_.]*:/s/:.*//p' <Makefile | sort
31 | 
32 | 
33 | release:
34 | 	git tag -f v$(version) && git push origin v$(version)
35 | 
36 | publish: clean-pyc clean-dist
37 | 	@echo "Create release $(version) and upload to pypi"
38 | 
39 | 	$(DOCKER_RUN_COMMAND) "python setup.py sdist bdist_wheel"
40 | 	$(DOCKER_RUN_COMMAND) 'for d in dist/* ; do twine register "$$d"; done;'
41 | 	$(DOCKER_RUN_COMMAND) "twine upload dist/*"
42 | 	@echo ""
43 | 
44 | test: clean-containers
45 | 	@echo "Test application $(version)"
46 | 	$(DOCKER_RUN_COMMAND) "uwsgi --pyrun setup.py --pyargv test --sharedarea=100 --enable-threads"
47 | 	@echo ""
48 | 
49 | tox: clean-containers
50 | 	@echo "Tox test application $(version)"
51 | 	$(DOCKER_RUN_COMMAND) "tox"
52 | 	@echo ""
53 | 
54 | lint: clean-containers
55 | 	@echo "Linting python files"
56 | 	$(DOCKER_RUN_COMMAND) "PYFLAKES_NODOCTEST=1 flake8 pyprometheus" || exit 1
57 | 	@echo ""
58 | 
59 | .PHONY: test publish lint help clean-pyc tox
60 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | Prometheus instrumentation library for Python applications
  2 | ============================================================
  3 | 
  4 | The unofficial Python 2 and 3 client for `Prometheus`_.
  5 | 
  6 | .. image:: https://travis-ci.org/Lispython/pyprometheus.svg?branch=master
  7 |     :target: https://travis-ci.org/Lispython/pyprometheus
  8 | 
  9 | 
 10 | 
 11 | Features
 12 | --------
 13 | 
 14 | - Four types of metric are supported: Counter, Gauge, Summary(without quantiles) and Histogram.
 15 | - InMemoryStorage (do not use it for multiprocessing apps)
 16 | - UWSGI storage - share metrics between processes
 17 | - UWAGI flush storage - sync metrics with uwsgi sharedarea by flush call
 18 | - time decorator
 19 | - time context manager
 20 | 
 21 | 
 22 | 
 23 | INSTALLATION
 24 | ------------
 25 | 
 26 | To use pyprometheus use pip or easy_install:
 27 | 
 28 | :code:`pip install pyprometheus`
 29 | 
 30 | or
 31 | 
 32 | :code:`easy_install pyprometheus`
 33 | 
 34 | 
 35 | HOW TO INSTRUMENTING CODE
 36 | -------------------------
 37 | 
 38 | Gauge
 39 | ~~~~~
 40 | 
 41 | A gauge is a metric that represents a single numerical value that can arbitrarily go up and down.::
 42 | 
 43 |    from pyprometheus import Gauge
 44 |    from pyprometheus import BaseRegistry, LocalMemoryStorage
 45 | 
 46 |    storage = LocalMemoryStorage()
 47 |    registry = CollectorRegistry(storage=storage)
 48 |    gauge = Gauge("job_in_progress", "Description", registry=registry)
 49 | 
 50 |    gauge.inc(10)
 51 |    gauge.dec(5)
 52 |    gauge.set(21.1)
 53 | 
 54 | 
 55 | utilities::
 56 | 
 57 |   gauge.set_to_current_time()   # Set to current unixtime
 58 | 
 59 |   # Increment when entered, decrement when exited.
 60 |   @gauge.track_in_progress()
 61 |   def f():
 62 |       pass
 63 | 
 64 |   with gauge.track_in_progress():
 65 |       pass
 66 | 
 67 | 
 68 |   with gauge.time():
 69 |       time.sleep(10)
 70 | 
 71 | 
 72 | 
 73 | Counter
 74 | ~~~~~~~
 75 | 
 76 | A counter is a cumulative metric that represents a single numerical value that only ever goes up.::
 77 | 
 78 |    from pyprometheus import Counter
 79 |    from pyprometheus import BaseRegistry, LocalMemoryStorage
 80 | 
 81 |    storage = LocalMemoryStorage()
 82 |    registry = CollectorRegistry(storage=storage)
 83 |    counter = Counter("requests_total", "Description", registry=registry)
 84 | 
 85 |    counter.inc(10)
 86 | 
 87 | 
 88 | 
 89 | 
 90 | 
 91 | Summary
 92 | ~~~~~~~
 93 | 
 94 | Similar to a histogram, a summary samples observations (usually things like request durations and response sizes).::
 95 | 
 96 |    from pyprometheus import Summary
 97 |    from pyprometheus import BaseRegistry, LocalMemoryStorage
 98 | 
 99 |    storage = LocalMemoryStorage()
100 |    registry = CollectorRegistry(storage=storage)
101 |    s = Summary("requests_duration_seconds", "Description", registry=registry)
102 | 
103 |    s.observe(0.100)
104 | 
105 | 
106 | utilities for timing code::
107 | 
108 |    @gauge.time()
109 |    def func():
110 |       time.sleep(10)
111 | 
112 |    with gauge.time():
113 |       time.sleep(10)
114 | 
115 | 
116 | 
117 | Histogram
118 | ~~~~~~~~~
119 | 
120 | A histogram samples observations (usually things like request durations or response sizes) and counts them in configurable buckets. It also provides a sum of all observed values.::
121 | 
122 |   from pyprometheus import Summary
123 |    from pyprometheus import BaseRegistry, LocalMemoryStorage
124 | 
125 |    storage = LocalMemoryStorage()
126 |    registry = CollectorRegistry(storage=storage)
127 |    histogram = Histogram("requests_duration_seconds", "Description", registry=registry)
128 | 
129 |    histogram.observe(1.1)
130 | 
131 | utilities for timing code::
132 | 
133 |    @histogram.time()
134 |    def func():
135 |       time.sleep(10)
136 | 
137 |    with histogram.time():
138 |       time.sleep(10)
139 | 
140 | 
141 | 
142 | Labels
143 | ~~~~~~
144 | 
145 | All metrics can have labels, allowing grouping of related time series.
146 | 
147 | 
148 | Example::
149 | 
150 |     from pyprometheus import Counter
151 |     c = Counter('my_requests_total', 'HTTP Failures', ['method', 'endpoint'])
152 |     c.labels('get', '/').inc()
153 |     c.labels('post', '/submit').inc()
154 | 
155 | or labels as keyword arguments::
156 | 
157 |     from pyprometheus import Counter
158 |     c = Counter('my_requests_total', 'HTTP Failures', ['method', 'endpoint'])
159 |     c.labels(method='get', endpoint='/').inc()
160 |     c.labels(method='post', endpoint='/submit').inc()
161 | 
162 | 
163 | 
164 | STORAGES
165 | --------
166 | 
167 | Currently library support 2 storages: LocalMemoryStorage and UWSGIStorage
168 | 
169 | Every registry MUST have link to storage::
170 | 
171 |   from pyprometheus import BaseRegistry, LocalMemoryStorage
172 | 
173 |   storage = LocalMemoryStorage()
174 |   registry = CollectorRegistry(storage=storage)
175 | 
176 | 
177 | Use LocalMemoryStorage
178 | ~~~~~~~~~~~~~~~~~~~~~~
179 | 
180 | Simple storage that store samples to application memory. It can be used with threads.::
181 | 
182 |   from pyprometheus import BaseRegistry, LocalMemoryStorage
183 | 
184 |   storage = LocalMemoryStorag()
185 | 
186 | 
187 | Use UWSGIStorage
188 | ~~~~~~~~~~~~~~~~
189 | 
190 | UWSGIStorage allow to use `uwsgi sharedarea`_ to sync metrics between processes.::
191 | 
192 |   from pyprometheus.contrib.uwsgi_features import UWSGICollector, UWSGIStorage
193 | 
194 |   SHAREDAREA_ID = 0
195 |   storage = UWSGIStorage(SHAREDAREA_ID)
196 | 
197 | 
198 | 
199 | also need to configure UWSGI sharedaread pages.
200 | 
201 | 
202 | 
203 | 
204 | EXPORTING
205 | ---------
206 | 
207 | Library have some helpers to export metrics
208 | 
209 | To text format
210 | ~~~~~~~~~~~~~~
211 | 
212 | You can convert registry to text format::
213 | 
214 | 
215 |   from pyprometheus import BaseRegistry, LocalMemoryStorage
216 |   from pyprometheus.utils.exposition import registry_to_text
217 |   from pyprometheus import Gauge
218 | 
219 |   storage = LocalMemoryStorage()
220 |   registry = CollectorRegistry(storage=storage)
221 |   g = Gauge('raid_status', '1 if raid array is okay', registry=registry)
222 |   g.set(1)
223 |   print(registry_to_text(registry))
224 | 
225 | 
226 | 
227 | Text file export
228 | ~~~~~~~~~~~~~~~~
229 | 
230 | This is useful for monitoring cronjobs, or for writing cronjobs to expose metrics about a machine system.::
231 | 
232 |   from pyprometheus import BaseRegistry, LocalMemoryStorage
233 |   from pyprometheus.utils.exposition import registry_to_text, write_to_textfile
234 |   from pyprometheus import Gauge
235 | 
236 |   storage = LocalMemoryStorage()
237 |   registry = CollectorRegistry(storage=storage)
238 |   g = Gauge('raid_status', '1 if raid array is okay', registry=registry)
239 |   g.set(1)
240 |   write_to_textfile(registry, "/path/to/file/metrics.prom")
241 | 
242 | 
243 | You can configure `text file collector`_ to use generated file.
244 | 
245 | 
246 | TODO
247 | ----
248 | 
249 | Some features that we plan to do:
250 | 
251 | - [ ] Add mmap storage
252 | - [ ] Add features for async frameworks
253 | - [ ] Optimize UWSGI storage byte pad
254 | - [ ] Add quantiles
255 | 
256 | 
257 | 
258 | EXAMPLE PROJECT
259 | ---------------
260 | 
261 | We create `example project`_ to show hot to use pyprometheus in real project.
262 | 
263 | 
264 | CONTRIBUTE
265 | ----------
266 | 
267 | Fork https://github.com/Lispython/pyprometheus/ , create commit and pull request to ``develop``.
268 | 
269 | 
270 | 
271 | .. _`example project`: http://github.com/Lispython/pyprometheus_demo
272 | .. _`text file collector`: https://github.com/prometheus/node_exporter#textfile-collector
273 | .. _`uwsgi sharedarea`: http://uwsgi-docs.readthedocs.io/en/latest/SharedArea.html
274 | .. _`Prometheus`: http://prometheus.io
275 | 


--------------------------------------------------------------------------------
/TODO.org:
--------------------------------------------------------------------------------
 1 | #+AUTHOR: Alexandr
 2 | #+CATEGORY: libs
 3 | #+STARTUP: hidestars
 4 | #+STARTUP: overview
 5 | #+TAGS: core(c) api(a) test(t) doc(d)
 6 | #+DRAWERS: HIDDEN STATE ORIGINAL LOG
 7 | 
 8 | 
 9 | 
10 | * [0/2][0%]
11 | ** TODO Metric name validation                                          :api:
12 | ** TODO Labels validation                                              :core:
13 | 


--------------------------------------------------------------------------------
/ci/setup:
--------------------------------------------------------------------------------
1 | #!/bin/bash -e
2 | 
3 | pip install "pip>=8.1"
4 | 
5 | pip install "uwsgi==$UWSGI"
6 | 
7 | pip install -U -r tests_requirements.txt
8 | 


--------------------------------------------------------------------------------
/ci/test:
--------------------------------------------------------------------------------
1 | #!/bin/bash -e
2 | 
3 | uwsgi --pyrun setup.py --pyargv test --sharedarea=100 --enable-threads
4 | 


--------------------------------------------------------------------------------
/conftest.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | import os
 4 | 
 5 | import pytest
 6 | import uwsgi
 7 | import time
 8 | from pyprometheus.storage import BaseStorage
 9 | from pyprometheus.utils import measure_time as measure_time_manager
10 | try:
11 |     xrange = xrange
12 | except Exception:
13 |     xrange = range
14 | 
15 | 
16 | @pytest.fixture
17 | def project_root():
18 |     return os.path.dirname(os.path.abspath(__file__))
19 | 
20 | 
21 | @pytest.yield_fixture(autouse=True)
22 | def run_around_tests():
23 |     m = uwsgi.sharedarea_memoryview(0)
24 |     for x in xrange(len(m)):
25 |         m[x] = "\x00"
26 | 
27 |     yield
28 | 
29 | 
30 | @pytest.fixture
31 | def measure_time():
32 |     return measure_time_manager
33 | 
34 | 
35 | @pytest.fixture()
36 | def iterations():
37 |     return 500
38 | 
39 | 
40 | @pytest.fixture()
41 | def num_workers():
42 |     return 10
43 | 


--------------------------------------------------------------------------------
/docs/usage.rst:
--------------------------------------------------------------------------------
1 | HOW TO INSTRUMENTING CODE
2 | =========================
3 | 


--------------------------------------------------------------------------------
/examples/test1.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | 
 5 | from pyprometheus.registry import BaseRegistry
 6 | 
 7 | registry = BaseRegistry()
 8 | 
 9 | 
10 | 
11 | total_requests = registry.gauge("app:total_requests",
12 |                                 "Documentation string", ["env_name"]).add_to_registry(register)
13 | 
14 | 
15 | total_request.labels(env_name="test").inc()
16 | 
17 | 
18 | response_time = registry.Histogram('name', 'Doc', ['label1'], buckets=(.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, float('inf')))
19 | 


--------------------------------------------------------------------------------
/pyprometheus/__init__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus
 5 | ~~~~~~~~~~~~
 6 | 
 7 | Prometheus instrumentation library for Python applications
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | 
14 | 
15 | __all__ = ("__version__", "__version_info__", "__maintainer__",
16 |            "Counter", "Gauge", "Summary", "Histogram", "BaseStorage",
17 |            "LocalMemoryStorage")
18 | 
19 | __license__ = "BSD, see LICENSE for more details"
20 | 
21 | __version__ = "0.0.9"
22 | 
23 | __version_info__ = list(map(int, __version__.split(".")))
24 | 
25 | __maintainer__ = "Alexandr Lispython"
26 | 
27 | from pyprometheus.metrics import Counter, Gauge, Summary, Histogram # noqa
28 | from pyprometheus.storage import BaseStorage, LocalMemoryStorage # noqa
29 | 


--------------------------------------------------------------------------------
/pyprometheus/compat.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus.compat
 5 | ~~~~~~~~~~~~~~~~~~~
 6 | 
 7 | Prometheus instrumentation library for Python applications
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | 
14 | import sys
15 | 
16 | # Useful for very coarse version differentiation.
17 | PY2 = sys.version_info[0] == 2
18 | PY3 = sys.version_info[0] == 3
19 | PY34 = sys.version_info[0:2] >= (3, 4)
20 | 


--------------------------------------------------------------------------------
/pyprometheus/const.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus.const
 5 | ~~~~~~~~~~~~~~~~~~
 6 | 
 7 | Prometheus instrumentation library for Python applications
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | 
14 | 
15 | class Types(object):
16 | 
17 |     BASE = 1
18 |     GAUGE = 2
19 |     COUNTER = 3
20 | 
21 |     SUMMARY = 4
22 |     SUMMARY_SUM = 5
23 |     SUMMARY_COUNTER = 7
24 |     SUMMARY_QUANTILE = 8
25 | 
26 |     HISTOGRAM = 10
27 | 
28 |     HISTOGRAM_SUM = 11
29 |     HISTOGRAM_COUNTER = 12
30 |     HISTOGRAM_BUCKET = 13
31 | 
32 | 
33 | TYPES = Types()
34 | 
35 | 
36 | CONTENT_TYPE = 'text/plain; version=0.0.4; charset=utf-8'
37 | 
38 | 
39 | CREDITS = """# Python client for prometheus.io
40 | # http://github.com/Lispython/pyprometheus
41 | # Generated at {dt}
42 | """
43 | 


--------------------------------------------------------------------------------
/pyprometheus/contrib/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Lispython/pyprometheus/d364b0dd01597b001102b5c8157f289f0eb18f5a/pyprometheus/contrib/__init__.py


--------------------------------------------------------------------------------
/pyprometheus/contrib/graphite.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus.contrib.graphite
 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 6 | 
 7 | Bridge to push metrics over TCP in the Graphite plaintext format.
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | 
14 | class GraphitePusher(object):
15 | 
16 |     def __init__(self, address, registry, connection_timeout=30):
17 |         self._connection_timeout = connection_timeout
18 |         self._address = address
19 |         self._registry = registry
20 | 
21 |     def format_sample(self, sample):
22 |         """Format single sample to graphite format
23 |         """
24 |         raise NotImplementedError
25 | 
26 | 
27 |     def push(self):
28 |         """Push samples from registry to graphite
29 |         """
30 |         raise NotImplementedError
31 | 


--------------------------------------------------------------------------------
/pyprometheus/contrib/uwsgi_features.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | """
  4 | pyprometheus.contrib.uwsgi_features
  5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  6 | 
  7 | UWSGI server process collector and storage
  8 | 
  9 | :copyright: (c) 2017 by Alexandr Lispython.
 10 | :license: , see LICENSE for more details.
 11 | :github: http://github.com/Lispython/pyprometheus
 12 | """
 13 | 
 14 | import marshal
 15 | import os
 16 | import struct
 17 | import uuid
 18 | import copy
 19 | from contextlib import contextmanager
 20 | from logging import getLogger
 21 | from pyprometheus.const import TYPES
 22 | from pyprometheus.metrics import Gauge, Counter
 23 | from pyprometheus.storage import BaseStorage, LocalMemoryStorage
 24 | 
 25 | 
 26 | try:
 27 |     import uwsgi
 28 | except ImportError:
 29 |     uwsgi = None
 30 | 
 31 | try:
 32 |     xrange = xrange
 33 | except Exception:
 34 |     xrange = range
 35 | 
 36 | 
 37 | class InvalidUWSGISharedareaPagesize(Exception):
 38 |     pass
 39 | 
 40 | logger = getLogger("pyprometheus.uwsgi_features")
 41 | 
 42 | 
 43 | class UWSGICollector(object):
 44 |     """Grap UWSGI stats and export to prometheus
 45 |     """
 46 |     def __init__(self, namespace, labels={}):
 47 |         self._namespace = namespace
 48 |         self._labels = tuple(sorted(labels.items(), key=lambda x: x[0]))
 49 |         self._collectors = self.declare_metrics()
 50 | 
 51 |     @property
 52 |     def uid(self):
 53 |         return "uwsgi-collector:{0}".format(self._namespace)
 54 | 
 55 |     def get_samples(self):
 56 |         """Get uwsgi stats
 57 |         """
 58 |         for collector in self.collect():
 59 |             yield collector, collector.get_samples()
 60 | 
 61 |     @property
 62 |     def text_export_header(self):
 63 |         return "# UWSGI stats metrics"
 64 | 
 65 |     def metric_name(self, name):
 66 |         """Make metric name with namespace
 67 | 
 68 |         :param name:
 69 |         """
 70 |         return ":".join([self._namespace, name])
 71 | 
 72 |     def declare_metrics(self):
 73 |         return {
 74 |             "memory": Gauge(self.metric_name("uwsgi_memory_bytes"), "UWSGI memory usage in bytes", ("type",) + self._labels),
 75 |             "processes": Gauge(self.metric_name("processes_total"), "Number of UWSGI processes", self._labels),
 76 |             "worker_status": Gauge(self.metric_name("worker_status_totla"), "Current workers status", self._labels),
 77 |             "total_requests": Gauge(self.metric_name("requests_total"), "Total processed request", self._labels),
 78 |             "buffer_size": Gauge(self.metric_name("buffer_size_bytes"), "UWSGI buffer size in bytes", self._labels),
 79 |             "started_on": Gauge(self.metric_name("started_on"), "UWSGI started on timestamp", self._labels),
 80 |             "cores": Gauge(self.metric_name("cores"), "system cores", self._labels),
 81 | 
 82 | 
 83 |             "process:respawn_count": Gauge(self.metric_name("process:respawn_count"), "Process respawn count", ("id", ) + self._labels),
 84 |             "process:last_spawn": Gauge(self.metric_name("process:last_spawn"), "Process last spawn", ("id", ) + self._labels),
 85 |             "process:signals": Gauge(self.metric_name("process:signals"), "Process signals total", ("id", ) + self._labels),
 86 |             "process:avg_rt": Gauge(self.metric_name("process:avg_rt"), "Process average response time", ("id", ) + self._labels),
 87 |             "process:tx": Gauge(self.metric_name("process:tx"), "Process transmitted data", ("id",) + self._labels),
 88 | 
 89 |             "process:status": Gauge(self.metric_name("process:status"), "Process status", ("id", "status") + self._labels),
 90 |             "process:running_time": Gauge(self.metric_name("process:running_time"), "Process running time", ("id", ) + self._labels),
 91 |             "process:exceptions": Gauge(self.metric_name("process:exceptions"), "Process exceptions", ("id", ) + self._labels),
 92 |             "process:requests": Gauge(self.metric_name("process:requests"), "Process requests", ("id", ) + self._labels),
 93 |             "process:delta_requests": Gauge(self.metric_name("process:delta_requests"), "Process delta_requests", ("id", ) + self._labels),
 94 |             "process:rss": Gauge(self.metric_name("process:rss"), "Process rss memory", ("id", ) + self._labels),
 95 |             "process:vsz": Gauge(self.metric_name("process:vzs"), "Process vsz address space", ("id", ) + self._labels),
 96 |         }
 97 | 
 98 |     def collect(self):
 99 |         for name, value in [("processes", uwsgi.numproc),
100 |                             ("total_requests", uwsgi.total_requests()),
101 |                             ("buffer_size", uwsgi.buffer_size),
102 |                             ("started_on", uwsgi.started_on),
103 |                             ("cores", uwsgi.cores)]:
104 |             yield self.get_sample(name, value)
105 | 
106 |         yield self.get_memory_samples()
107 | 
108 |         for x in self.get_workers_samples(uwsgi.workers()):
109 |             yield x
110 | 
111 | 
112 |     def get_workers_samples(self, workers):
113 |         """Read worker stats and create samples
114 | 
115 |         :param worker: worker stats
116 |         """
117 |         for name in ["requests", "respawn_count", "running_time",
118 |                      "exceptions", "delta_requests",
119 |                      "rss", "vsz", "last_spawn", "tx", "avg_rt", "signals"]:
120 |             metric = self._collectors["process:" + name]
121 | 
122 |             for worker in workers:
123 |                 labels = self._labels + (("id", worker["id"]),)
124 |                 metric.add_sample(labels, metric.build_sample(labels,
125 |                                   (  (TYPES.GAUGE, metric.name, "", labels, worker[name]),  )))
126 | 
127 |             yield metric
128 | 
129 |         metric = self._collectors["process:status"]
130 |         for worker in workers:
131 |             labels = self._labels + (("id", worker["id"]), ("status", worker["status"]))
132 |             metric.add_sample(labels, metric.build_sample(labels,
133 |                                 (  (TYPES.GAUGE, metric.name, "", self._labels + (("id", worker["id"]), ("status", worker["status"])), 1),  )))
134 | 
135 |         yield metric
136 | 
137 |     def get_sample(self, name, value):
138 |         """Create sample for given name and value
139 | 
140 |         :param name:
141 |         :param value:
142 |         """
143 |         metric = self._collectors[name]
144 |         return metric.build_samples([(self._labels, (  (TYPES.GAUGE, metric.name, "", self._labels, float(value)),  ))])
145 | 
146 |     def get_memory_samples(self):
147 |         """Get memory usage samples
148 |         """
149 |         metric = self._collectors["memory"]
150 |         return metric.build_samples(
151 |             [(self._labels + (("type", "rss"),), (  (TYPES.GAUGE, metric.name, "", self._labels + (("type", "rss"),), uwsgi.mem()[0]),  )),
152 |              (self._labels + (("type", "vsz"),), (  (TYPES.GAUGE, metric.name, "", self._labels + (("type", "vsz"),), uwsgi.mem()[1]),  ))])
153 | 
154 | 
155 | class UWSGIStorage(BaseStorage):
156 |     """A dict of doubles, backend by uwsgi sharedarea"""
157 | 
158 |     SHAREDAREA_ID = int(os.environ.get("PROMETHEUS_UWSGI_SHAREDAREA", 0))
159 |     KEY_SIZE_SIZE = 4
160 |     KEY_VALUE_SIZE = 8
161 |     SIGN_SIZE = 10
162 |     AREA_SIZE_SIZE = 4
163 |     SIGN_POSITION = 4
164 |     AREA_SIZE_POSITION = 0
165 | 
166 |     def __init__(self, sharedarea_id=SHAREDAREA_ID, namespace="", stats=False, labels={}):
167 |         self._sharedarea_id = sharedarea_id
168 |         self._used = None
169 |         # Changed every time then keys added
170 |         self._sign = None
171 |         self._positions = {}
172 |         self._rlocked = False
173 |         self._wlocked = False
174 |         self._keys_cache = {}
175 |         self._namespace = namespace
176 |         self._stats = stats
177 |         self._labels = tuple(sorted(labels.items(), key=lambda x: x[0]))
178 | 
179 |         self._syncs = 0
180 | 
181 |         self._m = uwsgi.sharedarea_memoryview(self._sharedarea_id)
182 | 
183 |         self.init_memory()
184 | 
185 |         self._collectors = self.declare_metrics()
186 | 
187 |     @property
188 |     def uid(self):
189 |         return "uwsgi-storage:{0}".format(self._namespace)
190 | 
191 |     @property
192 |     def text_export_header(self):
193 |         return "# {0} stats metrics".format(self.__class__.__name__)
194 | 
195 |     def metric_name(self, name):
196 |         """Make metric name with namespace
197 | 
198 |         :param name:
199 |         """
200 |         return ":".join([self._namespace, name])
201 | 
202 | 
203 |     @staticmethod
204 |     def get_unique_id():
205 |         try:
206 |             return uwsgi.worker_id()
207 |         except Exception:
208 |             try:
209 |                 return uwsgi.mule_id()
210 |             except Exception:
211 |                 return os.getpid()
212 |         return "unknown"
213 | 
214 |     def declare_metrics(self):
215 |         return {
216 |             "memory_sync": Counter(self.metric_name("memory_read"), "UWSGI shared memory syncs", ("sharedarea", "id") + self._labels),
217 |             "memory_size": Gauge(self.metric_name("memory_size"), "UWSGI shared memory size", ("sharedarea", ) + self._labels),
218 |             "num_keys": Gauge(self.metric_name("num_keys"), "UWSGI num_keys", ("sharedarea", ) + self._labels)
219 |         }
220 | 
221 |     def collect(self):
222 |         labels = self._labels + (("sharedarea", self._sharedarea_id), ("id", self.get_unique_id()))
223 |         metric = self._collectors["memory_sync"]
224 |         metric.add_sample(labels, metric.build_sample(labels, (   (TYPES.GAUGE, metric.name, "", labels, self._syncs), )))
225 | 
226 |         yield metric
227 | 
228 |         labels = self._labels + (("sharedarea", self._sharedarea_id), )
229 | 
230 |         # yield metric
231 |         metric = self._collectors["memory_size"]
232 | 
233 |         metric.add_sample(labels, metric.build_sample(labels, (   (TYPES.GAUGE, metric.name, "", labels, self.get_area_size()), )))
234 | 
235 |         yield metric
236 | 
237 |         metric = self._collectors["num_keys"]
238 |         metric.add_sample(labels, metric.build_sample(labels, (   (TYPES.GAUGE, metric.name, "", labels, len(self._positions)), )))
239 | 
240 |         yield metric
241 | 
242 | 
243 |     @property
244 |     def m(self):
245 |         return self._m
246 | 
247 |     @property
248 |     def wlocked(self):
249 |         return self._wlocked
250 | 
251 |     @wlocked.setter
252 |     def wlocked(self, value):
253 |         self._wlocked = value
254 |         return self._wlocked
255 | 
256 |     @property
257 |     def rlocked(self):
258 |         return self._rlocked
259 | 
260 |     @rlocked.setter
261 |     def rlocked(self, value):
262 |         self._rlocked = value
263 |         return self._rlocked
264 | 
265 |     def serialize_key(self, key):
266 |         try:
267 |             return self._keys_cache[key]
268 |         except KeyError:
269 |             self._keys_cache[key] = val = marshal.dumps(key)
270 |             return val
271 | 
272 |     def unserialize_key(self, serialized_key):
273 |         if not serialized_key:
274 |             raise RuntimeError("Invalid serialized key")
275 |         return marshal.loads(serialized_key)
276 | 
277 |     def get_area_size_with_lock(self):
278 |         with self.lock():
279 |             return self.get_area_size()
280 | 
281 |     def get_slice(self, start, size):
282 |         return slice(start, start+size)
283 | 
284 |     def get_area_size(self):
285 |         """Read area size from uwsgi
286 |         """
287 |         return struct.unpack(b"i", self.m[self.get_slice(self.AREA_SIZE_POSITION, self.AREA_SIZE_SIZE)])[0]
288 | 
289 |     def init_area_size(self):
290 |         return self.update_area_size(self.AREA_SIZE_SIZE)
291 | 
292 |     def update_area_size(self, size):
293 |         self._used = size
294 |         self.m[self.get_slice(self.AREA_SIZE_POSITION, self.AREA_SIZE_SIZE)] = struct.pack(b"i", size)
295 |         return True
296 | 
297 |     def update_area_sign(self):
298 |         self._sign = os.urandom(self.SIGN_SIZE)
299 |         self.m[self.get_slice(self.SIGN_POSITION, self.SIGN_SIZE)] = self._sign
300 | 
301 | 
302 |     def get_area_sign(self):
303 |         """Get current area sign from memory
304 |         """
305 |         return self.m[self.get_slice(self.SIGN_POSITION, self.SIGN_SIZE)].tobytes()
306 | 
307 |     def init_memory(self, validation=True):
308 |         """Initialize default memory addresses
309 |         """
310 |         with self.lock():
311 |             if self._used is None:
312 |                 self._used = self.get_area_size()
313 | 
314 |             if self._used == 0:
315 |                 self.update_area_sign()
316 |                 self.update_area_size(self.SIGN_SIZE + self.AREA_SIZE_SIZE)
317 | 
318 |             if validation:
319 |                 self.validate_actuality()
320 | 
321 | 
322 |     def read_memory(self):
323 |         """Read all keys from sharedared
324 |         """
325 |         if self.get_area_size() == 0:
326 |             self.init_memory(False)
327 | 
328 |         pos = self.AREA_SIZE_POSITION + self.AREA_SIZE_SIZE + self.SIGN_SIZE
329 |         self._used = self.get_area_size()
330 |         self._sign = self.get_area_sign()
331 |         self._positions.clear()
332 | 
333 |         while pos < self._used + self.AREA_SIZE_POSITION:
334 | 
335 |             key_size, (key, key_value), positions = self.read_item(pos)
336 |             yield key_size, (key, key_value), positions
337 |             pos = positions[3]
338 | 
339 |     def load_exists_positions(self):
340 |         """Load all keys from memory
341 |         """
342 |         self._syncs += 1
343 |         self._used = self.get_area_size()
344 |         self._sign = self.get_area_sign()
345 | 
346 |         for _, (key, _), positions in self.read_memory():
347 |             self._positions[key] = positions
348 |             #self._keys_cache[marshal.loads(key)] = key
349 | 
350 |     def get_string_padding(self, key):
351 |         """Calculate string padding
352 | 
353 |         http://stackoverflow.com/questions/11642210/computing-padding-required-for-n-byte-alignment
354 |         :param key: encoded string
355 |         """
356 |         #return (4 - (len(key) % 4)) % 4
357 | 
358 |         return (8 - (len(key) + 4) % 8)
359 | 
360 |     def get_key_size(self, key):
361 |         """Calculate how many memory need key
362 |         :param key: key string
363 |         """
364 |         return len(self.serialize_key(key)) + self.KEY_SIZE_SIZE + self.KEY_VALUE_SIZE
365 | 
366 | 
367 |     def get_binary_string(self, key, value):
368 |         item_template = "=i{0}sd".format(len(key)).encode()
369 | 
370 |         return struct.pack(item_template, len(key), key, value)
371 | 
372 |     def init_key(self, key, init_value=0.0):
373 |         """Initialize memory for key
374 | 
375 |         :param key: key string
376 |         """
377 | 
378 |         value = self.get_binary_string(key, init_value)
379 | 
380 |         key_string_position = self._used + self.AREA_SIZE_POSITION
381 | 
382 |         self.m[self.get_slice(key_string_position, len(value))] = value
383 | 
384 |         self.update_area_size(self._used + len(value))
385 |         self._positions[key] = [key_string_position, key_string_position + self.KEY_SIZE_SIZE,
386 |                                 self._used - self.KEY_VALUE_SIZE, self._used]
387 |         self.update_area_sign()
388 |         return self._positions[key]
389 | 
390 |     def read_key_string(self, position, size):
391 |         """Read key value from position by given size
392 | 
393 |         :param position: int offset for key string
394 |         :param size:  int key size in bytes to read
395 |         """
396 |         key_string_bytes = self.m[self.get_slice(position, size)]
397 |         return struct.unpack(b"{0}s".format(size), key_string_bytes)[0]
398 | 
399 | 
400 |     def read_key_value(self, position):
401 |         """Read float value of position
402 | 
403 |         :param position: int offset for key value float
404 |         """
405 |         key_value_bytes = self.m[self.get_slice(position, self.KEY_VALUE_SIZE)]
406 |         return struct.unpack(b"d", key_value_bytes)[0]
407 | 
408 |     def read_key_size(self, position):
409 |         """Read key size from position
410 | 
411 |         :param position: int offset for 4-byte key size
412 |         """
413 |         key_size_bytes = self.m[self.get_slice(position, self.KEY_SIZE_SIZE)]
414 |         return struct.unpack(b"i", key_size_bytes)[0]
415 | 
416 |     def write_key_value(self, position, value):
417 |         """Write float value to position
418 | 
419 |         :param position: int offset for 8-byte float value
420 |         """
421 |         self.m[self.get_slice(position, self.KEY_VALUE_SIZE)] = struct.pack(b"d", value)
422 |         return value
423 | 
424 |     def read_item(self, position):
425 |         """Read key info from given position
426 | 
427 |         4 bytes int key size
428 |         n bytes key value of utf-8 encoded string key padding to a 8 byte
429 |         8 bytes float counter value
430 |         """
431 | 
432 |         key_size = self.read_key_size(position)
433 | 
434 |         key_string_position = position + self.KEY_SIZE_SIZE
435 | 
436 |         key = self.read_key_string(key_string_position, key_size)
437 | 
438 |         key_value_position = key_string_position + key_size  # + self.get_string_padding(key)
439 | 
440 |         key_value = self.read_key_value(key_value_position)
441 |         return (key_size,
442 |                 (key, key_value),
443 |                 (position, key_string_position,
444 |                  key_value_position, key_value_position + self.KEY_VALUE_SIZE))
445 | 
446 |     def get_key_position(self, key, init_value=0.0):
447 |         try:
448 |             return self._positions[key], False
449 |         except Exception:
450 |             return (self.init_key(key, init_value=init_value), True)
451 | 
452 |     def inc_value(self, key, value):
453 |         """Increase/decrease key value
454 | 
455 |         :param key: key string
456 |         :param value: key value
457 |         """
458 |         with self.lock():
459 |             try:
460 |                 self.validate_actuality()
461 |                 positions, created = self.get_key_position(self.serialize_key(key), value)
462 |                 if created:
463 |                     return value
464 |                 return self.write_key_value(positions[2], self.read_key_value(positions[2]) + value)
465 |             except InvalidUWSGISharedareaPagesize as e:
466 |                 logger.error("Invalid sharedarea pagesize {0} bytes".format(len(self._m)))
467 |                 return 0
468 |             except Exception as e:
469 |                 logger.error(e, exc_info=True)
470 |                 return 0
471 | 
472 |     def write_value(self, key, value):
473 |         """Write value to shared memory
474 | 
475 |         :param key: key string
476 |         :param value: key value
477 |         """
478 |         with self.lock():
479 |             try:
480 |                 self.validate_actuality()
481 |                 positions, created = self.get_key_position(self.serialize_key(key), value)
482 |                 if created:
483 |                     return value
484 |                 return self.write_key_value(positions[2], value)
485 |             except InvalidUWSGISharedareaPagesize as e:
486 |                 logger.error("Invalid sharedarea pagesize {0} bytes".format(len(self._m)))
487 |                 return None
488 |             except Exception as e:
489 |                 logger.error(e, exc_info=True)
490 |                 return 0
491 | 
492 |     def get_value(self, key):
493 |         """Read value from shared memory
494 | 
495 |         :param key: key string
496 |         """
497 |         with self.lock():
498 |             try:
499 |                 self.validate_actuality()
500 |                 return self.read_key_value(self.get_key_position(self.serialize_key(key))[0][2])
501 |             except InvalidUWSGISharedareaPagesize:
502 |                 logger.error("Invalid sharedarea pagesize {0} bytes".format(len(self._m)))
503 |                 return 0
504 |             except Exception as e:
505 |                 logger.error(e, exc_info=True)
506 |                 return 0
507 | 
508 |     @property
509 |     def is_actual(self):
510 |         return self._sign == self.get_area_sign()
511 | 
512 |     def validate_actuality(self):
513 |         """For prevent data corruption
514 | 
515 |         Reload data from sharedmemory into process if sign changed
516 |         """
517 |         if not self.is_actual:
518 |             self.load_exists_positions()
519 | 
520 |         return True
521 | 
522 |     @contextmanager
523 |     def lock(self):
524 |         lock_id = uuid.uuid4().hex
525 |         if not self.wlocked and not self.rlocked:
526 |             self.wlocked, self.rlocked = lock_id, lock_id
527 |             uwsgi.sharedarea_wlock(self._sharedarea_id)
528 |             try:
529 |                 yield
530 |             except Exception as e:
531 |                 logger.error(e, exc_info=True)
532 |             uwsgi.sharedarea_unlock(self._sharedarea_id)
533 |             self.wlocked, self.rlocked = False, False
534 |         else:
535 |             yield
536 | 
537 |     @contextmanager
538 |     def rlock(self):
539 |         lock_id = uuid.uuid4().hex
540 |         if not self.rlocked:
541 |             self.rlocked = lock_id
542 |             uwsgi.sharedarea_rlock(self._sharedarea_id)
543 |             try:
544 |                 yield
545 |             except Exception as e:
546 |                 logger.error(e, exc_info=True)
547 |             uwsgi.sharedarea_unlock(self._sharedarea_id)
548 |             self.rlocked = False
549 |         else:
550 |             yield
551 | 
552 |     def unlock(self):
553 |         self._wlocked, self._rlocked = False, False
554 |         uwsgi.sharedarea_unlock(self._sharedarea_id)
555 | 
556 |     def __len__(self):
557 |         return len(self._positions)
558 | 
559 |     def clear(self):
560 |         for x in xrange(self.AREA_SIZE_SIZE + self.AREA_SIZE_SIZE):
561 |             self.m[x] = "\x00"
562 | 
563 |         self._positions.clear()
564 | 
565 |     def get_items(self):
566 |         with self.rlock():
567 |             self.validate_actuality()
568 | 
569 |         for key, position in self._positions.items():
570 |             yield self.unserialize_key(key), self.read_key_value(position[2])
571 | 
572 |     def inc_items(self, items):
573 | 
574 |         with self.lock():
575 |             self.validate_actuality()
576 | 
577 |             for key, value in items:
578 |                 try:
579 |                     positions, created = self.get_key_position(self.serialize_key(key), value)
580 |                     if created:
581 |                         continue
582 |                     self.write_key_value(positions[2], self.read_key_value(positions[2]) + value)
583 |                 except InvalidUWSGISharedareaPagesize:
584 |                     logger.error("Invalid sharedarea pagesize {0} bytes".format(len(self._m)))
585 |                 except Exception as e:
586 |                     logger.error(e, exc_info=True)
587 |                     return 0
588 | 
589 |     def write_items(self, items):
590 | 
591 |         with self.lock():
592 |             self.validate_actuality()
593 | 
594 |             for key, value in items:
595 |                 try:
596 |                     positions, created = self.get_key_position(self.serialize_key(key), value)
597 |                     if created:
598 |                         continue
599 |                     self.write_key_value(positions[2], value)
600 |                 except InvalidUWSGISharedareaPagesize:
601 |                     logger.error("Invalid sharedarea pagesize {0} bytes".format(len(self._m)))
602 |                 except Exception as e:
603 |                     logger.error(e, exc_info=True)
604 |                     return 0
605 | 
606 | 
607 | class UWSGIFlushStorage(LocalMemoryStorage):
608 |     """Storage wrapper for UWSGI storage that update couters inmemory and flush into uwsgi sharedarea
609 |     """
610 |     SHAREDAREA_ID = int(os.environ.get("PROMETHEUS_UWSGI_SHAREDAREA", 0))
611 | 
612 |     def __init__(self, sharedarea_id=UWSGIStorage.SHAREDAREA_ID, namespace="", stats=False, labels={}):
613 |         self._uwsgi_storage = UWSGIStorage(sharedarea_id, namespace=namespace, stats=stats, labels=labels)
614 |         self._flush = 0
615 |         self._get_items = 0
616 |         self._clear = 0
617 |         super(UWSGIFlushStorage, self).__init__()
618 | 
619 |     @property
620 |     def persistent_storage(self):
621 |         return self._uwsgi_storage
622 | 
623 |     def flush(self):
624 |         items = list(super(UWSGIFlushStorage, self).get_items())
625 |         self._uwsgi_storage.inc_items(items)
626 |         super(UWSGIFlushStorage, self).clear()
627 | 
628 |     def get_items(self):
629 |         return self._uwsgi_storage.get_items()
630 | 
631 |     def __len__(self):
632 |         return super(UWSGIFlushStorage, self).__len__()
633 | 
634 |     def clear(self):
635 |         self._uwsgi_storage.clear()
636 |         super(UWSGIFlushStorage, self).clear()
637 | 


--------------------------------------------------------------------------------
/pyprometheus/managers.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | """
 5 | pyprometheus.values
 6 | ~~~~~~~~~~~~~~~~~~~
 7 | 
 8 | Prometheus instrumentation library for Python applications
 9 | 
10 | :copyright: (c) 2017 by Alexandr Lispython.
11 | :license: , see LICENSE for more details.
12 | :github: http://github.com/Lispython/pyprometheus
13 | """
14 | import time
15 | from functools import wraps
16 | 
17 | default_timer = time.time
18 | 
19 | class BaseManager(object):
20 |     def __call__(self, f):
21 |         @wraps(f)
22 |         def wrapper(*args, **kwargs):
23 |             with self:
24 |                 return f(*args, **kwargs)
25 |         return wrapper
26 | 
27 | 
28 | class TimerManager(BaseManager):
29 |     def __init__(self, collector):
30 |         self._collector = collector
31 | 
32 |     def __enter__(self):
33 |         self._start_time = default_timer()
34 | 
35 |     def __exit__(self, exc_type, exc_value, traceback):
36 |         self._collector.observe(default_timer() - self._start_time)
37 | 
38 | 
39 | class InprogressTrackerManager(BaseManager):
40 | 
41 |     def __init__(self, gauge):
42 |         self._gauge = gauge
43 | 
44 |     def __enter__(self):
45 |         self._gauge.inc()
46 | 
47 |     def __exit__(self, exc_info, exc_value, traceback):
48 |         self._gauge.dec()
49 | 
50 | 
51 | class GaugeTimerManager(TimerManager):
52 | 
53 |     def __exit__(self, exc_type, exc_value, traceback):
54 |         self._collector.set(default_timer() - self._start_time)
55 | 


--------------------------------------------------------------------------------
/pyprometheus/metrics.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | """
  4 | pyprometheus.metrics
  5 | ~~~~~~~~~~~~~~~~~~~~
  6 | 
  7 | Prometheus instrumentation library for Python applications
  8 | 
  9 | :copyright: (c) 2017 by Alexandr Lispython.
 10 | :license: , see LICENSE for more details.
 11 | :github: http://github.com/Lispython/pyprometheus
 12 | """
 13 | 
 14 | from pyprometheus.const import TYPES
 15 | from pyprometheus.utils import escape_str
 16 | from pyprometheus.values import (MetricValue, GaugeValue,
 17 |                                  CounterValue, SummaryValue,
 18 |                                  HistogramValue)
 19 | 
 20 | 
 21 | class BaseMetric(object):
 22 | 
 23 |     value_class = MetricValue
 24 | 
 25 |     NOT_ALLOWED_LABELS = set()
 26 | 
 27 |     TYPE = "untyped"
 28 | 
 29 |     PARENT_METHODS = set()
 30 | 
 31 |     def __init__(self, name, doc, labels=[], registry=None):
 32 |         self._name = name
 33 |         self._doc = doc
 34 |         self._labelnames = tuple(sorted(labels))
 35 |         self.validate_labelnames(labels)
 36 |         self._storage = None
 37 | 
 38 |         if registry is not None:
 39 |             self.add_to_registry(registry)
 40 | 
 41 |         self._samples = {}
 42 | 
 43 |         self._labels_cache = {}
 44 | 
 45 |     def __repr__(self):
 46 |         return u"<{0}[{1}]: {2} samples>".format(self.__class__.__name__, self._name, len(self._samples))
 47 | 
 48 |     def get_proxy(self):
 49 |         if self._labelnames:
 50 |             raise RuntimeError("You need to use labels")
 51 |         return self.value_class(self, label_values={})
 52 | 
 53 |     def validate_labelnames(self, names):
 54 |         for name in names:
 55 |             if name in self.NOT_ALLOWED_LABELS:
 56 |                 raise RuntimeError("Label name {0} not allowed for {1}".format(name, self.__class__.__name__))
 57 |         return True
 58 | 
 59 |     @property
 60 |     def name(self):
 61 |         return self._name
 62 | 
 63 |     @property
 64 |     def doc(self):
 65 |         return self._doc
 66 | 
 67 |     @property
 68 |     def label_names(self):
 69 |         return self._labelnames
 70 | 
 71 |     @property
 72 |     def uid(self):
 73 |         return "{0}-{1}".format(self._name, str(self._labelnames))
 74 | 
 75 |     def add_to_registry(self, registry):
 76 |         """Add metric to registry
 77 |         """
 78 |         registry.register(self)
 79 |         self._storage = registry.storage
 80 |         return self
 81 | 
 82 |     def labels(self, *args, **kwargs):
 83 |         if args and isinstance(args[0], dict):
 84 |             label_values = self.value_class.prepare_labels(args[0])[0]
 85 |         else:
 86 |             label_values = self.value_class.prepare_labels(kwargs)[0]
 87 |         return self._labels_cache.setdefault((label_values, self.value_class.TYPE),
 88 |                                              self.value_class(self, label_values=label_values))
 89 | 
 90 |     @property
 91 |     def text_export_header(self):
 92 |         """
 93 |         Format description lines for collector
 94 |         # HELP go_gc_duration_seconds A summary of the GC invocation durations.
 95 |         # TYPE go_gc_duration_seconds summary
 96 |         """
 97 |         return "\n".join(["# HELP {name} {doc}",
 98 |                           "# TYPE {name} {metric_type}"]).format(
 99 |                               name=escape_str(self.name),
100 |                               doc=escape_str(self.doc),
101 |                               metric_type=self.TYPE)
102 | 
103 |     def build_samples(self, items):
104 |         """Build samples from objects
105 | 
106 |         [((2, "metric_gauge_name", "", (("label1", "value3"), ("label2", "value4"))), 5.0)]
107 |         """
108 |         for label_values, data in items:
109 |             self.add_sample(label_values, self.build_sample(label_values, data))
110 |         return self
111 | 
112 |     def clear_samples(self):
113 |         self._samples.clear()
114 | 
115 |     def build_sample(self, label_values, item):
116 |         """Build value object from given data
117 |         """
118 |         return self.value_class(self, label_values=label_values, value=item[0][-1])
119 | 
120 | 
121 |     def add_sample(self, label_values, value):
122 |         self._samples[tuple(sorted(label_values, key=lambda x: x[0]))] = value
123 | 
124 |     def get_samples(self):
125 |         """Get samples from storage
126 |         """
127 |         return self._samples.values()
128 | 
129 | 
130 |     def __getattr__(self, name):
131 |         if name in self.PARENT_METHODS:
132 |             return getattr(self.get_proxy(), name)
133 | 
134 |         raise AttributeError
135 |         # return super(BaseMetric, self).__getattr__(name)
136 | 
137 | 
138 | class Gauge(BaseMetric):
139 | 
140 |     TYPE = "gauge"
141 | 
142 |     value_class = GaugeValue
143 | 
144 |     PARENT_METHODS = set(("inc", "dec", "set", "get", "track_inprogress",
145 |                         "set_to_current_time", "time", "value"))
146 | 
147 | 
148 | class Counter(BaseMetric):
149 |     TYPE = "counter"
150 | 
151 |     value_class = CounterValue
152 | 
153 |     PARENT_METHODS = set(("inc", "get", "value"))
154 | 
155 | 
156 | class Summary(BaseMetric):
157 | 
158 |     TYPE = "summary"
159 |     DEFAULT_QUANTILES = (0, 0.25, 0.5, 0.75, 1)
160 | 
161 |     value_class = SummaryValue
162 | 
163 |     NOT_ALLOWED_LABELS = set("quantile")
164 | 
165 |     PARENT_METHODS = set(("observe", "value", "time"))
166 | 
167 |     def __init__(self, name, doc, labels=[], quantiles=False, registry=None):
168 |         self._quantiles = list(sorted(quantiles)) if quantiles else []
169 |         super(Summary, self).__init__(name, doc, labels, registry)
170 | 
171 |     @property
172 |     def quantiles(self):
173 |         return self._quantiles
174 | 
175 |     def build_sample(self, label_values, data):
176 |         subtypes = {
177 |             "sum": None,
178 |             "count": None,
179 |             "quantiles": [] if isinstance(self._quantiles, (list, tuple)) else None
180 |         }
181 | 
182 |         for meta, value in data:
183 |             value_class = self.value_class.SUBTYPES[meta[2]]
184 | 
185 |             if meta[0] == TYPES.SUMMARY_SUM:
186 |                 subtypes["sum"] = value_class(self, label_values=label_values, value=value)
187 |             elif meta[0] == TYPES.SUMMARY_COUNTER:
188 |                 subtypes["count"] = value_class(self, label_values=label_values, value=value)
189 |             elif meta[0] == TYPES.SUMMARY_QUANTILE:
190 |                 quantile = dict(meta[3])["quantile"]
191 |                 subtypes["quantiles"].append(
192 |                     value_class(self, label_values=label_values, quantile=quantile, value=value))
193 | 
194 |         return self.value_class(self, label_values=label_values, value=subtypes)
195 | 
196 | 
197 | class Histogram(BaseMetric):
198 |     TYPE = "histogram"
199 | 
200 |     DEFAULT_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5,
201 |                        0.75, 1.0, 2.5, 5.0, 7.5, 10.0, float("inf"))
202 | 
203 |     NOT_ALLOWED_LABELS = set("le")
204 | 
205 |     value_class = HistogramValue
206 | 
207 |     PARENT_METHODS = set(("observe", "value", "time"))
208 | 
209 |     def __init__(self, name, doc, labels=[], buckets=DEFAULT_BUCKETS, registry=None):
210 |         self._buckets = list(sorted(buckets)) if buckets else []
211 |         super(Histogram, self).__init__(name, doc, labels, registry)
212 | 
213 |     @property
214 |     def buckets(self):
215 |         return self._buckets
216 | 
217 | 
218 |     def build_sample(self, label_values, data):
219 |         subtypes = {
220 |             "sum": None,
221 |             "count": None,
222 |             "buckets": [] if isinstance(self._buckets, (list, tuple)) else None
223 |         }
224 | 
225 |         for meta, value in data:
226 |             value_class = self.value_class.SUBTYPES[meta[2]]
227 | 
228 |             if meta[0] == TYPES.HISTOGRAM_SUM:
229 |                 subtypes["sum"] = value_class(self, label_values=label_values, value=value)
230 |             elif meta[0] == TYPES.HISTOGRAM_COUNTER:
231 |                 subtypes["count"] = value_class(self, label_values=label_values, value=value)
232 |             elif meta[0] == TYPES.HISTOGRAM_BUCKET:
233 |                 bucket = dict(meta[3])["bucket"]
234 |                 subtypes["buckets"].append(
235 |                     value_class(self, label_values=label_values, bucket=bucket, value=value))
236 | 
237 |         return self.value_class(self, label_values=label_values, value=subtypes)
238 | 


--------------------------------------------------------------------------------
/pyprometheus/registry.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus
 5 | ~~~~~~~~~~~~
 6 | 
 7 | Prometheus instrumentation library for Python applications
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | 
14 | 
15 | 
16 | class BaseRegistry(object):
17 |     """Link with metrics collectors
18 |     """
19 |     def __init__(self, storage={}):
20 |         self._collectors = {}
21 |         self._storage = storage
22 | 
23 |     @property
24 |     def storage(self):
25 |         return self._storage
26 | 
27 |     def register(self, collector):
28 |         """Add collector to registry
29 |         """
30 |         if collector.uid in self._collectors:
31 |             raise RuntimeError(u"Collector {0} already registered".format(collector.uid))
32 |         self._collectors[collector.uid] = collector
33 | 
34 |     def unregister(self, collector):
35 |         """Remove collector from registry
36 |         """
37 |         self._collectors.pop(collector.uid, None)
38 | 
39 |     def collect(self, clean=True):
40 |         """Get all metrics from all registered collectos
41 |         """
42 |         data = dict(self._storage.items())
43 | 
44 |         for uid, collector in self.collectors():
45 |             if clean and hasattr(collector, 'clear_samples'):
46 |                 collector.clear_samples()
47 | 
48 |             # Collectors with collect method already have stats
49 |             if hasattr(collector, 'collect'):
50 |                 for item in collector.collect():
51 |                     yield item
52 |             else:
53 |                 yield collector.build_samples(data.get(collector.name, []))
54 | 
55 |     def collectors(self):
56 |         return self._collectors.items()
57 | 
58 |     def is_registered(self, collector):
59 |         """Check that collector already exists
60 |         """
61 |         return collector.uid in self._collectors
62 | 
63 |     def __len__(self):
64 |         return len(self._collectors)
65 | 
66 |     def get_samples(self):
67 |         """
68 |         Yield collector object and its samples
69 |         """
70 |         for collector in self.collect():
71 |             yield collector, collector.get_samples()
72 | 


--------------------------------------------------------------------------------
/pyprometheus/scripts/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 | 
4 | def main():
5 |     print("Grab system info for prometheus")
6 | 


--------------------------------------------------------------------------------
/pyprometheus/storage.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | """
  4 | pyprometheus.storage
  5 | ~~~~~~~~~~~~~~~~~~~~
  6 | 
  7 | Prometheus instrumentation library for Python applications
  8 | 
  9 | :copyright: (c) 2017 by Alexandr Lispython.
 10 | :license: , see LICENSE for more details.
 11 | :github: http://github.com/Lispython/pyprometheus
 12 | """
 13 | 
 14 | 
 15 | from collections import defaultdict
 16 | from itertools import groupby
 17 | from threading import Lock
 18 | 
 19 | from pyprometheus.const import TYPES
 20 | 
 21 | 
 22 | class BaseStorage(object):
 23 | 
 24 |     def inc_value(self, key, amount):
 25 |         raise NotImplementedError("inc_value")
 26 | 
 27 |     def write_value(self, key, value):
 28 |         raise NotImplementedError("write_value")
 29 | 
 30 |     def get_value(self, key):
 31 |         raise NotImplementedError("get_value")
 32 | 
 33 |     def get_items(self):
 34 |         raise NotImplementedError("get_items")
 35 | 
 36 |     def __len__(self):
 37 |         raise NotImplementedError("len")
 38 | 
 39 |     def __repr__(self):
 40 |         return u"<{0}: {1} items>".format(self.__class__.__name__, len(self))
 41 | 
 42 |     def items(self):
 43 |         """Read all keys from storage and yield grouped by name metrics and their samples
 44 | 
 45 |         ((name1, ((labels1, data1), (labels2 data1)))
 46 |          (name2, ((labels1, data2), (labels2 data2))))
 47 | 
 48 |         """
 49 |         for name, items in groupby(sorted(self.get_items(), key=self.sorter),
 50 |                                    key=self.name_group):
 51 | 
 52 |             yield name, self.group_by_labels(items)
 53 | 
 54 |     def group_by_labels(self, items):
 55 |         """Group metric by labels
 56 |         """
 57 |         return [(k, list(v)) for k, v in groupby(items, key=self.label_group)]
 58 | 
 59 |     def name_group(self, value):
 60 |         """Group keys by metric name
 61 |         :param value: (type, name, subtype, labels dict, value)
 62 |         """
 63 |         return value[0][1]
 64 | 
 65 |     def sorter(self, value):
 66 |         """Sort keys by (name, labels, type)
 67 | 
 68 |         :param value: (type, name, subtype, labels dict, value)
 69 |         """
 70 |         if TYPES.HISTOGRAM_BUCKET == value[0][0]:
 71 |             return value[0][1], value[0][3][1:], value[0][0]
 72 |         return value[0][1], value[0][3], value[0][0]
 73 | 
 74 |     def label_group(self, value):
 75 |         """Group by labels
 76 |         :param value: (type, name, subtype, lebels dict, value)
 77 |         """
 78 |         if TYPES.HISTOGRAM_BUCKET == value[0][0]:
 79 |             return value[0][3][1:]
 80 |         return value[0][3]
 81 | 
 82 | 
 83 | class LocalMemoryStorage(BaseStorage):
 84 | 
 85 |     def __init__(self):
 86 |         self._storage = defaultdict(float)
 87 |         self._lock = Lock()
 88 | 
 89 |     def inc_value(self, key, value):
 90 |         with self._lock:
 91 |             self._storage[key] += value
 92 | 
 93 |     def write_value(self, key, value):
 94 |         with self._lock:
 95 |             self._storage[key] = value
 96 | 
 97 |     def get_value(self, key):
 98 |         with self._lock:
 99 |             return self._storage[key]
100 | 
101 |     def get_items(self):
102 |         return self._storage.items()
103 | 
104 |     def __len__(self):
105 |         return len(self._storage)
106 | 
107 |     def __repr__(self):
108 |         return u"<{0}: {1} items>".format(self.__class__.__name__, len(self))
109 | 
110 |     def clear(self):
111 |         """Remove all items from storage
112 |         """
113 |         self._storage.clear()
114 | 


--------------------------------------------------------------------------------
/pyprometheus/utils/__init__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus.utils
 5 | ~~~~~~~~~~~~~~~~~~
 6 | 
 7 | Prometheus instrumentation library for Python applications
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | import sys
14 | import time
15 | 
16 | 
17 | def import_storage(path):
18 |     try:
19 |         __import__(path)
20 |     except ImportError:
21 |         raise
22 |     else:
23 |         return sys.modules[path]
24 | 
25 | 
26 | def escape_str(value):
27 |     return value.replace("\\", r"\\").replace("\n", r"\n").replace("\"", r"\"")
28 | 
29 | 
30 | def format_binary(value):
31 |     return ":".join("{0}>{1}".format(i, x.encode("hex")) for i, x in enumerate(value))
32 | 
33 | 
34 | def format_char_positions(value):
35 |     return ":".join("{0}>{1}".format(i, x) for i, x in enumerate(value))
36 | 
37 | 
38 | def print_binary(value):
39 |     print(value)
40 |     print(format_char_positions(value))
41 |     print(format_binary(value))
42 | 
43 | 
44 | class measure_time(object):
45 |     def __init__(self,name):
46 |         self.name = name
47 |         self._num_ops = 0
48 | 
49 |     def __enter__(self):
50 |         self.start = time.time()
51 |         return self
52 | 
53 |     def __exit__(self,ty,val,tb):
54 |         end = time.time()
55 |         print("{0} : {1:.4f} seconds for {2} ops [{3:.4f} / s]".format(
56 |             self.name, end-self.start,
57 |             self._num_ops, self._num_ops / (end-self.start)))
58 |         return False
59 | 
60 |     def set_num_ops(self, value):
61 |         self._num_ops = value
62 | 


--------------------------------------------------------------------------------
/pyprometheus/utils/exposition.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus.utils.exposition
 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 6 | 
 7 | Helpers to export registry to another formats
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | import os
14 | from datetime import datetime
15 | 
16 | from pyprometheus.const import CREDITS
17 | 
18 | 
19 | def registry_to_text(registry):
20 |     """Get all registry metrics and convert to text format
21 |     """
22 |     output = [CREDITS.format(dt=datetime.utcnow().isoformat())]
23 |     for collector, samples in registry.get_samples():
24 |         output.append(collector.text_export_header)
25 |         for sample in samples:
26 |             output.append(sample.export_str)
27 |     output.append("")
28 |     return "\n".join(output)
29 | 
30 | 
31 | def write_to_textfile(registry, path):
32 |     """Write metrics to text file
33 |     """
34 | 
35 |     tmp_filename = "{0}.{1}.tmp".format(path, os.getpid())
36 | 
37 |     with open(tmp_filename, "wb") as f:
38 |         f.write(registry_to_text(registry))
39 | 
40 |     os.rename(tmp_filename, path)
41 | 


--------------------------------------------------------------------------------
/pyprometheus/values.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | """
  4 | pyprometheus.values
  5 | ~~~~~~~~~~~~~~~~~~~
  6 | 
  7 | Prometheus instrumentation library for Python applications
  8 | 
  9 | :copyright: (c) 2017 by Alexandr Lispython.
 10 | :license: , see LICENSE for more details.
 11 | :github: http://github.com/Lispython/pyprometheus
 12 | """
 13 | 
 14 | import time
 15 | 
 16 | from pyprometheus.utils import escape_str
 17 | from pyprometheus.const import TYPES
 18 | from pyprometheus.managers import TimerManager, InprogressTrackerManager, GaugeTimerManager
 19 | 
 20 | 
 21 | class MetricValue(object):
 22 |     """Base metric collector"""
 23 | 
 24 |     TYPE = TYPES.BASE
 25 |     POSTFIX = ""
 26 | 
 27 |     def __init__(self, metric, label_values={}, value=None):
 28 |         self._metric = metric
 29 |         self.validate_labels(metric.label_names, label_values)
 30 | 
 31 |         self._labels, self._label_values = self.prepare_labels(label_values)
 32 |         self._value = value
 33 | 
 34 |     @staticmethod
 35 |     def prepare_labels(label_values):
 36 |         if isinstance(label_values, (list, tuple)):
 37 |             labels = tuple(sorted(label_values, key=lambda x: x[0]))
 38 |         elif isinstance(label_values, dict):
 39 |             labels = tuple(sorted(label_values.items(), key=lambda x: x[0]))
 40 |         return labels, dict(label_values)
 41 | 
 42 |     @property
 43 |     def metric(self):
 44 |         return self._metric
 45 | 
 46 |     def set_value(self, value):
 47 |         self._value = value
 48 | 
 49 |     def __repr__(self):
 50 |         return u"<{0}[{1}]: {2} -> {3}>".format(
 51 |             self.__class__.__name__, self._metric.name,
 52 |             str(self._labels).replace("'", "\""), self.__repr_value__())
 53 | 
 54 |     def validate_labels(self, label_names, labels):
 55 |         if len(labels) != len(label_names):
 56 |             raise RuntimeError(u"Invalid label values size: {0} != {1}".format(
 57 |                 len(label_names), len(labels)))
 58 | 
 59 |     def __repr_value__(self):
 60 |         return self.get()
 61 | 
 62 |     # def __str__(self):
 63 |     #     return u"{0}{1}".format(self.__class__.__name__, self._labels)
 64 | 
 65 |     @property
 66 |     def key(self):
 67 |         return (self.TYPE, self._metric.name, self.POSTFIX, self._labels)
 68 | 
 69 |     def inc(self, amount=1):
 70 |         return self._metric._storage.inc_value(self.key, amount)
 71 | 
 72 |     def get(self):
 73 |         # Do not lookup storage if value 0
 74 |         if self._value is not None:
 75 |             return self._value
 76 |         return self._metric._storage.get_value(self.key)
 77 | 
 78 |     @property
 79 |     def value(self):
 80 |         return self.get()
 81 | 
 82 |     @property
 83 |     def export_str(self):
 84 |         return "{name}{postfix}{{{labels}}} {value} {timestamp}".format(
 85 |             name=escape_str(self._metric.name), postfix=self.POSTFIX,
 86 |             labels=self.export_labels, timestamp=int(time.time() * 1000), value=float(self.value))
 87 | 
 88 |     @property
 89 |     def export_labels(self):
 90 |         return ", ".join(["{0}=\"{1}\"".format(self.format_export_label(name), self.format_export_value(value))
 91 |                           for name, value in self._labels])
 92 | 
 93 |     def format_export_label(self, label):
 94 |         if label == "bucket":
 95 |             return "le"
 96 |         return escape_str(label)
 97 | 
 98 |     def format_export_value(self, value):
 99 |         if value == float("inf"):
100 |             return "+Inf"
101 |         elif value == float("-inf"):
102 |             return "-Inf"
103 |         # elif math.isnan(value):
104 |         #     return "NaN"
105 |         return escape_str(str(value))
106 | 
107 | 
108 | class GaugeValue(MetricValue):
109 | 
110 |     TYPE = TYPES.GAUGE
111 | 
112 |     def dec(self, amount=1):
113 |         self.inc(-amount)
114 | 
115 |     def set(self, value):
116 |         self._metric._storage.write_value(self.key, value)
117 |         return value
118 | 
119 |     @property
120 |     def value(self):
121 |         return self.get()
122 | 
123 |     def track_in_progress(self):
124 |         return InprogressTrackerManager(self)
125 | 
126 |     def set_to_current_time(self):
127 |         return self.set(time.time())
128 | 
129 |     def time(self):
130 |         return GaugeTimerManager(self)
131 | 
132 | 
133 | class CounterValue(MetricValue):
134 | 
135 |     TYPE = TYPES.COUNTER
136 | 
137 |     @property
138 |     def value(self):
139 |         return self.get()
140 | 
141 | 
142 | class SummarySumValue(CounterValue):
143 |     TYPE = TYPES.SUMMARY_SUM
144 |     POSTFIX = "_sum"
145 | 
146 | 
147 | class SummaryCountValue(CounterValue):
148 |     TYPE = TYPES.SUMMARY_COUNTER
149 |     POSTFIX = "_count"
150 | 
151 | 
152 | class SummaryQuantilyValue(GaugeValue):
153 |     TYPE = TYPES.SUMMARY_QUANTILE
154 | 
155 |     POSTFIX = "_quantile"
156 | 
157 |     def __init__(self, metric, label_values={}, quantile=0, value=None):
158 |         label_values = dict(label_values).copy()
159 |         label_values["quantile"] = quantile
160 |         self._quantile = quantile
161 |         super(SummaryQuantilyValue, self).__init__(metric, label_values, value)
162 | 
163 |     def validate_labels(self, label_names, labels):
164 |         if len(labels) != len(label_names) + 1:
165 |             raise RuntimeError(u"Invalid label values size: {0} != {1}".format(
166 |                 len(label_names), len(labels) + 1))
167 | 
168 |     def __repr_value__(self):
169 |         return u"{0} -> {1}".format(self._quantile, self._value)
170 | 
171 |     @property
172 |     def key(self):
173 |         return (self.TYPE, self._metric.name, self.POSTFIX, self._labels)
174 |         # return (self.TYPE, self._metric.name, self._metric.name, self._labels)
175 | 
176 | 
177 | class SummaryValue(MetricValue):
178 |     u"""
179 |     summary with a base metric name of <basename> exposes multiple time series during a scrape:
180 | 
181 |      streaming φ-quantiles (0 ≤ φ ≤ 1) of observed events, exposed as <basename>{quantile="<φ>"}
182 |      the total sum of all observed values, exposed as <basename>_sum
183 |      the count of events that have been observed, exposed as <basename>_count
184 |     """
185 | 
186 |     TYPE = TYPES.SUMMARY
187 | 
188 |     SUBTYPES = {
189 |         "_sum": SummarySumValue,
190 |         "_count": SummaryCountValue,
191 |         "_quantile": SummaryQuantilyValue
192 |     }
193 | 
194 |     def __init__(self, metric, label_values={}, value={}):
195 | 
196 |         super(SummaryValue, self).__init__(metric, label_values=label_values)
197 |         self._sum = value.pop("sum", None) or SummarySumValue(self._metric, label_values=self._label_values)
198 |         self._count = value.pop("count", None) or SummaryCountValue(self._metric, label_values=self._label_values)
199 |         if isinstance(self._metric.quantiles, (list, tuple)):
200 | 
201 |             self._quantiles = value.pop("quantiles", []) or [SummaryQuantilyValue(self._metric, label_values=self._label_values, quantile=quantile)
202 |                                                              for quantile in self._metric.quantiles]
203 |         else:
204 |             self._quantiles = []
205 | 
206 |     def __repr_value__(self):
207 |         return u"sum={sum} / count={count} = {value} [{quantiles}]".format(
208 |             **{
209 |                 "sum": self._sum.value,
210 |                 "count": self._count.value,
211 |                 "value": (self._sum.value / self._count.value) if self._count.value != 0 else "-",
212 |                 "quantiles": ", ".join([x.__repr_value__() for x in self._quantiles]) if self._quantiles else "empty"
213 |             }
214 |         )
215 | 
216 |     def observe(self, amount):
217 |         self._sum.inc(amount)
218 |         self._count.inc()
219 | 
220 |         # TODO: calculate quantiles
221 |         # for quantile, value in self._quantiles:
222 |         #     pass
223 | 
224 |     @property
225 |     def value(self):
226 |         return {
227 |             "sum": self._sum,
228 |             "count": self._count,
229 |             "quantiles": self._quantiles}
230 | 
231 |     @property
232 |     def export_str(self):
233 |         return "\n".join([self._sum.export_str, self._count.export_str] + [quantile.export_str for quantile in self._quantiles])
234 | 
235 |     def time(self):
236 |         return TimerManager(self)
237 | 
238 | 
239 | class HistogramCountValue(SummaryCountValue):
240 |     TYPE = TYPES.HISTOGRAM_COUNTER
241 |     POSTFIX = "_count"
242 | 
243 | 
244 | class HistogramSumValue(SummarySumValue):
245 |     TYPE = TYPES.HISTOGRAM_SUM
246 |     POSTFIX = "_sum"
247 | 
248 | 
249 | class HistogramBucketValue(SummaryCountValue):
250 |     """
251 |     """    """
252 |     <basename>_bucket{le="<upper inclusive bound>"}
253 |     """
254 |     POSTFIX = "_bucket"
255 |     TYPE = TYPES.HISTOGRAM_BUCKET
256 | 
257 |     def __init__(self, metric, label_values={}, bucket=None, value=None):
258 |         label_values = dict(label_values).copy()
259 |         label_values["bucket"] = bucket
260 |         self._bucket_threshold = bucket
261 |         super(HistogramBucketValue, self).__init__(metric, label_values, value)
262 | 
263 |     def __repr_value__(self):
264 |         return u"{0} -> {1}".format(self._bucket_threshold, self._value)
265 | 
266 |     @property
267 |     def bucket_threshold(self):
268 |         return self._bucket_threshold
269 | 
270 |     def validate_labels(self, label_names, labels):
271 |         if len(labels) != len(label_names) + 1:
272 |             raise RuntimeError(u"Invalid label values size: {0} != {1}".format(
273 |                 len(label_names), len(labels) + 1))
274 | 
275 | 
276 | class HistogramValue(MetricValue):
277 |     TYPE = TYPES.HISTOGRAM
278 | 
279 |     SUBTYPES = {
280 |         "_sum": HistogramSumValue,
281 |         "_count": HistogramCountValue,
282 |         "_bucket": HistogramBucketValue
283 |     }
284 | 
285 |     def __init__(self, metric, label_values={}, value={}):
286 |         self._buckets = []
287 |         super(HistogramValue, self).__init__(metric, label_values=label_values)
288 | 
289 |         self._sum = value.pop("sum", None) or HistogramSumValue(self._metric, label_values=self._label_values)
290 |         self._count = value.pop("count", None) or HistogramCountValue(self._metric, label_values=self._label_values)
291 | 
292 |         self._buckets = (value.pop("buckets", []) or [HistogramBucketValue(self._metric, label_values=self._label_values, bucket=bucket)
293 |                                                       for bucket in sorted(self._metric.buckets)])
294 | 
295 |     def __repr_value__(self):
296 |         return u"sum={sum} / count={count} = {value} [{buckets}]".format(
297 |             **{
298 |                 "sum": self._sum.__repr_value__(),
299 |                 "count": self._count.__repr_value__(),
300 |                 "value": (self._sum.value / self._count.value) if self._count.value != 0 else "-",
301 |                 # "buckets": ""
302 |                 "buckets": ", ".join([x.__repr_value__() for x in self._buckets]) if self._buckets else "empty"
303 |             }
304 |         )
305 | 
306 |     def observe(self, amount):
307 |         self._sum.inc(amount)
308 |         self._count.inc()
309 | 
310 |         for bucket in self._buckets:
311 |             bucket.inc(int(amount < bucket.bucket_threshold))
312 | 
313 |     @property
314 |     def value(self):
315 |         return {
316 |             "sum": self._sum,
317 |             "count": self._count,
318 |             "buckets": self._buckets
319 |         }
320 | 
321 |     @property
322 |     def export_str(self):
323 |         return "\n".join([self._sum.export_str, self._count.export_str] + [bucket.export_str for bucket in self._buckets])
324 | 
325 |     def time(self):
326 |         return TimerManager(self)
327 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
 1 | [tool:pytest]
 2 | python_files=test_*.py
 3 | testpaths = tests
 4 | addopts=-s -p no:doctest --flake8 --cov=./
 5 | norecursedirs=pyprometheus build bin dist docs .git
 6 | flake8-max-line-length = 100
 7 | flake8-ignore =
 8 |     *.py E501
 9 |     main/settings/*.py F403 F401
10 |     */migrations/* ALL
11 | 
12 | [flake8]
13 | ignore = E501,F403,F401,D100,D101,D102,D103,I004,I001,I003,Q000,D205,D400,D105
14 | max-line-length = 100
15 | exclude = .tox,.git,docs,.ropeproject
16 | inline-quotes = double
17 | 
18 | 
19 | [bdist_wheel]
20 | universal = 1
21 | 
22 | [coverage:run]
23 | omit =
24 |      conftest.py
25 |      *test_*.py
26 |      *tests_*.py
27 |      fabfile.py
28 |      setup.py
29 |      .eggs/*
30 |      .tox/*
31 | 
32 | [coverage:report]
33 | # Regexes for lines to exclude from consideration
34 | exclude_lines =
35 |     # Have to re-enable the standard pragma
36 |     pragma: no cover
37 | 
38 | 
39 | [metadata]
40 | description-file = README.rst


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | """
 4 | pyprometheus
 5 | ~~~~~~~~~~~~
 6 | 
 7 | Prometheus instrumentation library for Python applications
 8 | 
 9 | :copyright: (c) 2017 by Alexandr Lispython <lispython@users.noreply.github.com>.
10 | :license: , see LICENSE for more details.
11 | :github: http://github.com/Lispython/pyprometheus
12 | """
13 | 
14 | 
15 | from setuptools import setup, find_packages
16 | from setuptools.command.test import test as TestCommand
17 | import re
18 | import sys
19 | import ast
20 | 
21 | _version_re = re.compile(r'__version__\s+=\s+(.*)')
22 | 
23 | with open('pyprometheus/__init__.py', 'rb') as f:
24 |     version = str(ast.literal_eval(_version_re.search(
25 |         f.read().decode('utf-8')).groups()[0]))
26 | 
27 | install_require = []
28 | tests_require = [x.strip() for x in open("tests_requirements.txt").readlines() if (x.strip() and not x.strip().startswith('#'))]
29 | 
30 | 
31 | def read_description():
32 |     try:
33 |         with open("README.rst", 'r') as f:
34 |             return f.read()
35 |     except Exception:
36 |         return __doc__
37 | 
38 | class PyTest(TestCommand):
39 | 
40 |     def initialize_options(self):
41 |         TestCommand.initialize_options(self)
42 |         self.pytest_args = []
43 | 
44 |     def finalize_options(self):
45 |         TestCommand.finalize_options(self)
46 |         self.test_args = []
47 |         self.test_suite = True
48 | 
49 |     def run_tests(self):
50 |         # import here, cause outside the eggs aren't loaded
51 |         import pytest
52 |         errno = pytest.main(self.pytest_args)
53 |         sys.exit(errno)
54 | 
55 | 
56 | setup(
57 |     name='pyprometheus',
58 |     version=version,
59 |     author='Alexandr Lispython',
60 |     author_email='lispython@users.noreply.github.com',
61 |     url='https://github.com/Lispython/pyprometheus',
62 |     description='Prometheus python client and instrumentation library',
63 |     long_description=read_description(),
64 |     packages=find_packages(exclude=("tests", "tests.*",)),
65 |     zip_safe=False,
66 |     extras_require={
67 |         'tests': tests_require,
68 |     },
69 |     license='BSD',
70 |     tests_require=tests_require,
71 |     install_requires=install_require,
72 |     cmdclass={'test': PyTest},
73 |     include_package_data=True,
74 |     entry_points={
75 |         'console_scripts': [
76 |             'pyprometheus = pyprometheus.scripts:main',
77 |         ]
78 |     },
79 |     classifiers=[
80 |         'Intended Audience :: Developers',
81 |         'Intended Audience :: System Administrators',
82 |         'Operating System :: OS Independent',
83 |         'Operating System :: POSIX',
84 |         'Programming Language :: Python :: 2',
85 |         'Programming Language :: Python :: 2.7',
86 |         # 'Programming Language :: Python :: 3',
87 |         # 'Programming Language :: Python :: 3.3',
88 |         # 'Programming Language :: Python :: 3.4',
89 |         # 'Programming Language :: Python :: 3.5',
90 |         'Programming Language :: Python',
91 |         'Programming Language :: Python :: Implementation :: CPython',
92 | 
93 |         'Topic :: Software Development',
94 |         'Topic :: System :: Monitoring',
95 |         'Topic :: Software Development :: Libraries',
96 |         'Topic :: System :: Networking :: Monitoring'
97 |     ],
98 | )
99 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 | 


--------------------------------------------------------------------------------
/tests/test_core.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 | 


--------------------------------------------------------------------------------
/tests/test_metrics.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | import pytest
  4 | import time
  5 | from pyprometheus.metrics import BaseMetric, Gauge, Counter, Histogram, Summary
  6 | from pyprometheus.registry import BaseRegistry
  7 | from pyprometheus.values import MetricValue
  8 | from pyprometheus.storage import LocalMemoryStorage
  9 | from pyprometheus.contrib.uwsgi_features import UWSGIStorage
 10 | 
 11 | 
 12 | @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage])
 13 | def test_base_metric(storage_cls):
 14 |     storage = storage_cls()
 15 |     registry = BaseRegistry(storage=storage)
 16 |     metric_name = "test_base_metric\x00\\"
 17 |     metric = BaseMetric(metric_name, "test_base_metric doc \u4500", ("label1", "label2"), registry=registry)
 18 | 
 19 |     assert registry.is_registered(metric)
 20 |     assert repr(metric) == "<BaseMetric[test_base_metric\x00\\]: 0 samples>"
 21 | 
 22 |     with pytest.raises(RuntimeError) as exc_info:
 23 |         registry.register(metric)
 24 | 
 25 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
 26 | 
 27 |     with pytest.raises(RuntimeError) as exc_info:
 28 |         metric.add_to_registry(registry)
 29 | 
 30 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
 31 | 
 32 |     labels = metric.labels({"label1\\x\n\"": "label1_value\\x\n\"", "label2": "label2_value\\x\n\""})
 33 | 
 34 |     assert isinstance(labels, MetricValue)
 35 | 
 36 |     labels.inc(1)
 37 | 
 38 |     assert labels.get() == 1
 39 | 
 40 |     assert metric.text_export_header == "\n".join(["# HELP test_base_metric\x00\\\\ test_base_metric doc \\\\u4500",
 41 |                                                    "# TYPE test_base_metric\x00\\\\ untyped"])
 42 | 
 43 |     assert labels.export_str.split(" ")[:2] == 'test_base_metric\x00\\\\{label1\\\\x\\n\\"="label1_value\\\\x\\n\\"", label2="label2_value\\\\x\\n\\""} 1.0'.split(" ")[:2] # noqa
 44 | 
 45 | 
 46 | @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage])
 47 | def test_counter_metric(storage_cls):
 48 |     storage = storage_cls()
 49 | 
 50 |     registry = BaseRegistry(storage=storage)
 51 |     metric_name = "counter_metric_name"
 52 |     metric = Counter(metric_name, "counter_metric_name doc", ("label1", "label2"), registry=registry)
 53 | 
 54 |     with pytest.raises(RuntimeError) as exc_info:
 55 |         metric.inc()
 56 |         assert exc_info.value == "You need to use labels"
 57 | 
 58 |     assert registry.is_registered(metric)
 59 | 
 60 |     assert repr(metric) == u"<Counter[counter_metric_name]: 0 samples>"
 61 | 
 62 |     with pytest.raises(RuntimeError) as exc_info:
 63 |         registry.register(metric)
 64 | 
 65 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
 66 | 
 67 |     with pytest.raises(RuntimeError) as exc_info:
 68 |         metric.add_to_registry(registry)
 69 | 
 70 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
 71 | 
 72 |     labels = metric.labels({"label1": "label1_value", "label2": "label2_value"})
 73 | 
 74 |     assert labels.get() == 0
 75 | 
 76 |     labels.inc(10)
 77 | 
 78 |     assert labels.get() == 10
 79 | 
 80 |     assert repr(labels) == str(labels)
 81 | 
 82 |     assert str(labels) == "<CounterValue[counter_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 10.0>"
 83 | 
 84 |     assert labels.key == (labels.TYPE, metric_name, labels.POSTFIX,
 85 |                           (("label1", "label1_value"), ("label2", "label2_value")))
 86 | 
 87 |     assert metric.text_export_header == "\n".join(["# HELP counter_metric_name counter_metric_name doc",
 88 |                                                    "# TYPE counter_metric_name counter"])
 89 | 
 90 | 
 91 | def test_gauge_metric():
 92 |     storage = LocalMemoryStorage()
 93 | 
 94 |     registry = BaseRegistry(storage=storage)
 95 |     metric_name = "gauge_metric_name"
 96 |     metric = Gauge(metric_name, metric_name + " doc", ("label1", "label2"), registry=registry)
 97 |     assert registry.is_registered(metric)
 98 | 
 99 |     with pytest.raises(RuntimeError) as exc_info:
100 |         metric.inc(10)
101 |         assert exc_info.value == "You need to use labels"
102 | 
103 |     assert repr(metric) == "<Gauge[gauge_metric_name]: 0 samples>"
104 | 
105 |     with pytest.raises(RuntimeError) as exc_info:
106 |         registry.register(metric)
107 | 
108 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
109 | 
110 |     with pytest.raises(RuntimeError) as exc_info:
111 |         metric.add_to_registry(registry)
112 | 
113 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
114 | 
115 |     labels = metric.labels({"label1": "label1_value", "label2": "label2_value"})
116 | 
117 |     assert labels.get() == 0
118 | 
119 |     labels.inc(10)
120 | 
121 |     assert labels.get() == 10
122 | 
123 |     assert repr(labels) == str(labels)
124 |     assert str(labels) == "<GaugeValue[gauge_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 10.0>"
125 | 
126 |     assert labels.key == (labels.TYPE, metric_name, labels.POSTFIX,
127 |                           (("label1", "label1_value"), ("label2", "label2_value")))
128 | 
129 |     assert metric.text_export_header == "\n".join(["# HELP gauge_metric_name gauge_metric_name doc",
130 |                                                    "# TYPE gauge_metric_name gauge"])
131 | 
132 |     with metric.labels({"label1": "1", "label2": "1"}).time():
133 | 
134 |         time.sleep(1)
135 | 
136 |     assert metric.labels(label1="1", label2="1").value > 1
137 | 
138 |     labels = metric.labels({"label1": "inprogress", "label2": "inprogress"})
139 | 
140 |     with labels.track_in_progress():
141 |         assert labels.value == 1
142 | 
143 |     assert labels.value == 0
144 | 
145 |     assert labels.set_to_current_time() == labels.value
146 | 
147 |     labels = metric.labels({"label1": "time2", "label2": "time2"})
148 | 
149 |     @labels.time()
150 |     def f(*args, **kwargs):
151 |         time.sleep(1)
152 | 
153 |     f()
154 |     assert labels.value > 1
155 | 
156 | 
157 | @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage])
158 | def test_summary(storage_cls):
159 |     storage = storage_cls()
160 | 
161 |     registry = BaseRegistry(storage=storage)
162 |     metric_name = "summary_metric_name"
163 |     metric = Summary(metric_name, "summary_metric_name doc", ("label1", "label2"), registry=registry)
164 | 
165 |     assert registry.is_registered(metric)
166 | 
167 |     with pytest.raises(RuntimeError) as exc_info:
168 |         metric.observe(10)
169 |         assert exc_info.value == "You need to use labels"
170 | 
171 |     assert repr(metric) == u"<Summary[summary_metric_name]: 0 samples>"
172 | 
173 |     with pytest.raises(RuntimeError) as exc_info:
174 |         registry.register(metric)
175 | 
176 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
177 | 
178 |     with pytest.raises(RuntimeError) as exc_info:
179 |         metric.add_to_registry(registry)
180 | 
181 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
182 | 
183 |     labels = metric.labels({"label1": "label1_value", "label2": "label2_value"})
184 | 
185 |     labels.observe(10)
186 | 
187 |     value = labels.value
188 | 
189 |     assert value["sum"].value == 10
190 |     assert value["count"].value == 1
191 | 
192 |     labels.observe(14)
193 | 
194 |     assert value["sum"].value == 24
195 |     assert value["count"].value == 2
196 | 
197 |     assert value["quantiles"] == []
198 | 
199 |     assert str(value["sum"]) == "<SummarySumValue[summary_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 24.0>"
200 |     assert str(value["count"]) == "<SummaryCountValue[summary_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 2.0>"
201 | 
202 |     assert value["sum"].key == (value["sum"].TYPE, "summary_metric_name", value["sum"].POSTFIX, (("label1", "label1_value"), ("label2", "label2_value")))
203 |     assert value["count"].key == (value["count"].TYPE, "summary_metric_name", value["count"].POSTFIX, (("label1", "label1_value"), ("label2", "label2_value")))
204 | 
205 |     assert metric.text_export_header == "\n".join(["# HELP summary_metric_name summary_metric_name doc",
206 |                                                    "# TYPE summary_metric_name summary"])
207 | 
208 |     for x in range(3):
209 |         with metric.labels({"label1": "1", "label2": "1"}).time():
210 | 
211 |             time.sleep(1)
212 | 
213 |     value = metric.labels(label1="1", label2="1").value
214 | 
215 |     assert value["sum"].value > 3
216 |     assert value["count"].value == 3
217 | 
218 |     labels = metric.labels({"label1": "time2", "label2": "time2"})
219 | 
220 |     @labels.time()
221 |     def f(*args, **kwargs):
222 |         time.sleep(1)
223 | 
224 |     for x in range(3):
225 |         f()
226 | 
227 |     value = labels.value
228 |     assert value["sum"].value > 3
229 |     assert value["count"].value == 3
230 | 
231 | 
232 | @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage])
233 | def test_histogram(storage_cls):
234 |     storage = storage_cls()
235 | 
236 |     registry = BaseRegistry(storage=storage)
237 |     metric_name = "histogram_metric_name"
238 |     metric = Histogram(metric_name, "histogram_metric_name doc", ("label1", "label2"), registry=registry)
239 | 
240 |     with pytest.raises(RuntimeError) as exc_info:
241 |         metric.observe(10)
242 |         assert exc_info.value == "You need to use labels"
243 | 
244 |     assert repr(metric) == u"<Histogram[histogram_metric_name]: 0 samples>"
245 | 
246 |     with pytest.raises(RuntimeError) as exc_info:
247 |         registry.register(metric)
248 | 
249 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
250 | 
251 |     with pytest.raises(RuntimeError) as exc_info:
252 |         metric.add_to_registry(registry)
253 | 
254 |         assert str(exc_info.value) == u"Collector {0} already registered.".format(metric.uid)
255 | 
256 |     labels = metric.labels({"label1": "label1_value", "label2": "label2_value"})
257 |     labels.observe(2.4)
258 | 
259 |     value = labels.value
260 | 
261 |     assert value["sum"].value == 2.4
262 |     assert value["count"].value == 1
263 | 
264 |     assert str(value["sum"]) == "<HistogramSumValue[histogram_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 2.4>"
265 |     assert str(value["count"]) == "<HistogramCountValue[histogram_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 1.0>"
266 | 
267 |     labels.observe(0.06)
268 | 
269 |     assert str(value["sum"]) == "<HistogramSumValue[histogram_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 2.46>"
270 |     assert str(value["count"]) == "<HistogramCountValue[histogram_metric_name]: ((\"label1\", \"label1_value\"), (\"label2\", \"label2_value\")) -> 2.0>"
271 | 
272 |     buckets = {x.bucket_threshold: x for x in value["buckets"]}
273 |     assert buckets[0.025].value == 0
274 |     assert buckets[0.075].value == 1
275 |     assert buckets[2.5].value == 2
276 |     assert buckets[float("inf")].value == 2
277 | 
278 |     assert value["sum"].key == (value["sum"].TYPE, "histogram_metric_name", value["sum"].POSTFIX, (("label1", "label1_value"), ("label2", "label2_value")))
279 |     assert value["count"].key == (value["count"].TYPE, "histogram_metric_name", value["count"].POSTFIX, (("label1", "label1_value"), ("label2", "label2_value")))
280 | 
281 |     assert metric.text_export_header == "\n".join(["# HELP histogram_metric_name histogram_metric_name doc",
282 |                                                    "# TYPE histogram_metric_name histogram"])
283 | 
284 |     for x in range(3):
285 |         with metric.labels({"label1": "1", "label2": "1"}).time():
286 | 
287 |             time.sleep(1)
288 | 
289 |     value = metric.labels(label1="1", label2="1").value
290 | 
291 |     assert value["sum"].value > 3
292 |     assert value["count"].value == 3
293 | 
294 |     labels = metric.labels({"label1": "time2", "label2": "time2"})
295 | 
296 |     @labels.time()
297 |     def f(*args, **kwargs):
298 |         time.sleep(1)
299 | 
300 |     for x in range(3):
301 |         f()
302 | 
303 |     value = labels.value
304 |     assert value["sum"].value > 3
305 |     assert value["count"].value == 3
306 | 
307 | 
308 | @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage])
309 | def test_metric_methods(storage_cls):
310 |     storage = storage_cls()
311 | 
312 |     registry = BaseRegistry(storage=storage)
313 | 
314 |     metric = Gauge("gauge_metric_name", "gauge_metric_name doc", registry=registry)
315 | 
316 |     metric.inc(2)
317 | 
318 |     assert metric.value == 2
319 | 
320 |     metric.dec(1)
321 | 
322 |     assert metric.value == 1
323 | 
324 |     assert metric.set(10) == 10
325 | 
326 |     assert metric.get() == 10
327 | 
328 |     with metric.time():
329 |         time.sleep(10)
330 | 
331 |     assert metric.value > 10
332 | 
333 |     assert metric.set_to_current_time() == metric.value
334 | 
335 |     metric = Counter("counter_metric_name", "counter_metric_name doc", registry=registry)
336 |     metric.inc(11)
337 | 
338 |     assert metric.value == 11
339 | 
340 |     assert metric.get() == 11
341 | 
342 |     metric = Summary("summary_metric_name", "summary_metric_name doc", registry=registry)
343 | 
344 |     for x in range(3):
345 |         with metric.time():
346 | 
347 |             time.sleep(1)
348 | 
349 |     assert metric.value["sum"].value > 3
350 |     assert metric.value["count"].value == 3
351 | 
352 |     metric = Histogram("histogram_metric_name", "histogram_metric_name doc", registry=registry)
353 | 
354 |     for x in range(3):
355 |         with metric.time():
356 | 
357 |             time.sleep(1)
358 | 
359 |     assert metric.value["sum"].value > 3
360 |     assert metric.value["count"].value == 3
361 | 


--------------------------------------------------------------------------------
/tests/test_registry.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | import pytest
  5 | 
  6 | from pyprometheus.registry import BaseRegistry
  7 | from pyprometheus.metrics import BaseMetric, Gauge, Counter, Histogram, Summary
  8 | from pyprometheus.storage import LocalMemoryStorage
  9 | from pyprometheus.contrib.uwsgi_features import UWSGIStorage
 10 | from pyprometheus.utils.exposition import registry_to_text, write_to_textfile
 11 | 
 12 | 
 13 | CONTROL_EXPORT = """
 14 | # HELP metric_counter_name doc_counter
 15 | # TYPE metric_counter_name counter
 16 | metric_counter_name{label1="value1", label2="value2"} 7.0 1487933466491
 17 | metric_counter_name{label1="value3", label2="value4"} 7.0 1487933466491
 18 | # HELP metric_summary_name doc_summary
 19 | # TYPE metric_summary_name summary
 20 | metric_summary_name_sum{label1="value1", label2="value2"} 4.0 1487933466491
 21 | metric_summary_name_count{label1="value1", label2="value2"} 1.0 1487933466491
 22 | metric_summary_name_sum{label1="value3", label2="value4"} 4.0 1487933466491
 23 | metric_summary_name_count{label1="value3", label2="value4"} 1.0 1487933466492
 24 | # HELP metric_untyped_name doc_untyped
 25 | # TYPE metric_untyped_name untyped
 26 | # HELP metric_histogram_name doc_histogram
 27 | # TYPE metric_histogram_name histogram
 28 | metric_histogram_name_sum{label1="value1", label2="value2"} 6.0 1487933466492
 29 | metric_histogram_name_count{label1="value1", label2="value2"} 1.0 1487933466492
 30 | metric_histogram_name_bucket{le="0.005", label1="value1", label2="value2"} 0.0 1487933466492
 31 | metric_histogram_name_bucket{le="0.01", label1="value1", label2="value2"} 0.0 1487933466493
 32 | metric_histogram_name_bucket{le="7.5", label1="value1", label2="value2"} 1.0 1487933466494
 33 | metric_histogram_name_bucket{le="+Inf", label1="value1", label2="value2"} 1.0 1487933466494
 34 | metric_histogram_name_sum{label1="value3", label2="value4"} 6.0 1487933466494
 35 | metric_histogram_name_count{label1="value3", label2="value4"} 1.0 1487933466494
 36 | metric_histogram_name_bucket{le="0.005", label1="value3", label2="value4"} 0.0 1487933466494
 37 | metric_histogram_name_bucket{le="0.01", label1="value3", label2="value4"} 0.0 1487933466495
 38 | metric_histogram_name_bucket{le="+Inf", label1="value3", label2="value4"} 1.0 1487933466496
 39 | metric_histogram_name_bucket{le="7.5", label1="value3", label2="value4"} 1.0 1487933466496
 40 | # HELP metric_gauge_name doc_gauge
 41 | # TYPE metric_gauge_name gauge
 42 | metric_gauge_name{label1="value1", label2="value2"} 5.0 1487933466497
 43 | metric_gauge_name{label1="value3", label2="value4"} 5.0 1487933466497"""
 44 | 
 45 | 
 46 | @pytest.mark.parametrize("storage_cls", [LocalMemoryStorage, UWSGIStorage])
 47 | def test_base_registry(storage_cls, measure_time):
 48 |     storage = storage_cls()
 49 |     registry = BaseRegistry(storage=storage)
 50 | 
 51 |     assert registry.storage == storage
 52 | 
 53 |     name_template = "metric_{0}_name"
 54 |     doc_template = "doc_{0}"
 55 |     metrics = {}
 56 |     labels = ("label1", "label2")
 57 |     labelnames = ("value1", "value2")
 58 | 
 59 |     for metric_class in [
 60 |             BaseMetric,
 61 |             Counter,
 62 |             Gauge,
 63 |             Summary]:
 64 |         metrics[metric_class.TYPE] = metric_class(
 65 |             name_template.format(metric_class.TYPE),
 66 |             doc_template.format(metric_class.TYPE),
 67 |             labels, registry=registry)
 68 | 
 69 |     metrics[Histogram.TYPE] = Histogram(
 70 |         name_template.format(Histogram.TYPE),
 71 |         doc_template.format(Histogram.TYPE),
 72 |         labels,
 73 |         buckets=(0.005, 0.01, 7.5, float("inf")),
 74 |         registry=registry
 75 |     )
 76 | 
 77 |     for k, v in metrics.items():
 78 |         assert registry.is_registered(v)
 79 | 
 80 |         registry.unregister(v)
 81 | 
 82 |         assert not registry.is_registered(v)
 83 | 
 84 |     for k, v in metrics.items():
 85 |         assert not registry.is_registered(v)
 86 | 
 87 |         registry.register(v)
 88 | 
 89 |         assert registry.is_registered(v)
 90 | 
 91 |     assert len(registry) == 5
 92 |     assert len(registry.collectors()) == 5
 93 | 
 94 |     labels_dict = dict(zip(labels, labelnames))
 95 | 
 96 |     metrics["gauge"].labels(**labels_dict).inc(5)
 97 |     metrics["counter"].labels(**labels_dict).inc(7)
 98 |     metrics["summary"].labels(**labels_dict).observe(4)
 99 |     metrics["histogram"].labels(**labels_dict).observe(6)
100 | 
101 |     labelnames2 = ("value3", "value4")
102 |     labels_dict2 = dict(zip(labels, labelnames2))
103 | 
104 |     metrics["gauge"].labels(**labels_dict2).inc(5)
105 |     metrics["counter"].labels(**labels_dict2).inc(7)
106 |     metrics["summary"].labels(**labels_dict2).observe(4)
107 |     metrics["histogram"].labels(**labels_dict2).observe(6)
108 | 
109 |     assert len(list(registry.get_samples())) == 5
110 | 
111 |     write_to_textfile(registry, "/tmp/metrics.prom")
112 | 
113 |     lines = []
114 | 
115 |     with open("/tmp/metrics.prom") as f:
116 |         for x in f:
117 |             if x:
118 |                 lines.append(x.strip())
119 | 
120 |     with measure_time("registry to text"):
121 | 
122 |         for test1, test2 in zip(registry_to_text(registry).split("\n")[4:], lines[4:]):
123 |             if test1.startswith("#"):
124 |                 assert test1 == test2
125 |             else:
126 |                 assert test1.split()[:-1] == test2.split()[:-1]
127 | 
128 |     metrics_count = map(lambda x: x.split(" ")[2],
129 |                         filter(lambda x: x.startswith("# HELP"), [x for x in registry_to_text(registry).split("\n")]))
130 | 
131 |     assert len(metrics_count) == len(set(metrics_count))
132 | 


--------------------------------------------------------------------------------
/tests/test_storage.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | from pyprometheus.storage import BaseStorage, LocalMemoryStorage
  4 | import random
  5 | import threading
  6 | 
  7 | try:
  8 |     xrange = xrange
  9 | except Exception:
 10 |     xrange = range
 11 | 
 12 | DATA = (
 13 |     ((2, "metric_gauge_name", "", (("label1", "value1"), ("label2", "value2"))), 5),
 14 |     ((3, "metric_counter_name", "", (("label1", "value1"), ("label2", "value2"))), 7),
 15 |     ((5, "metric_summary_name", "_sum", (("label1", "value1"), ("label2", "value2"))), 4),
 16 |     ((7, "metric_summary_name", "_count", (("label1", "value1"), ("label2", "value2"))), 1),
 17 |     ((11, "metric_histogram_name", "_sum", (("label1", "value1"), ("label2", "value2"))), 6),
 18 |     ((12, "metric_histogram_name", "_count", (("label1", "value1"), ("label2", "value2"))), 1),
 19 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 0.005), ("label1", "value1"), ("label2", "value2"))), 0),
 20 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 0.01), ("label1", "value1"), ("label2", "value2"))), 0),
 21 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 7.5), ("label1", "value1"), ("label2", "value2"))), 1),
 22 |     ((13, "metric_histogram_name", "_bucket", (("bucket", float("inf")), ("label1", "value1"), ("label2", "value2"))), 1),
 23 |     ((2, "metric_gauge_name", "", (("label1", "value3"), ("label2", "value4"))), 5),
 24 |     ((3, "metric_counter_name", "", (("label1", "value3"), ("label2", "value4"))), 7),
 25 |     ((5, "metric_summary_name", "_sum", (("label1", "value3"), ("label2", "value4"))), 4),
 26 |     ((7, "metric_summary_name", "_count", (("label1", "value3"), ("label2", "value4"))), 1),
 27 |     ((11, "metric_histogram_name", "_sum", (("label1", "value3"), ("label2", "value4"))), 6),
 28 |     ((12, "metric_histogram_name", "_count", (("label1", "value3"), ("label2", "value4"))), 1),
 29 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 0.005), ("label1", "value3"), ("label2", "value4"))), 0),
 30 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 0.01), ("label1", "value3"), ("label2", "value4"))), 0),
 31 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 7.5), ("label1", "value3"), ("label2", "value4"))), 1),
 32 |     ((13, "metric_histogram_name", "_bucket", (("bucket", float("inf")), ("label1", "value3"), ("label2", "value4"))), 1))
 33 | 
 34 | 
 35 | def test_base_storage():
 36 |     storage = BaseStorage()
 37 | 
 38 |     assert isinstance(storage, BaseStorage)
 39 | 
 40 | 
 41 | def test_local_memory_storage():
 42 |     storage = LocalMemoryStorage()
 43 | 
 44 |     assert len(storage) == 0
 45 | 
 46 |     key1 = (1,
 47 |             "metric_name1",
 48 |             "",
 49 |             (("key1", "value1"),
 50 |              ("key2", "value2")))
 51 | 
 52 |     key2 = (1,
 53 |             "metric_name2",
 54 |             "",
 55 |             (("key1", "value1"),
 56 |              ("key2", "value2")))
 57 | 
 58 |     storage.inc_value(key1, 1)
 59 |     assert storage.get_value(key1) == 1.0
 60 | 
 61 |     storage.inc_value(key2, 4)
 62 |     assert storage.get_value(key2) == 4.0
 63 | 
 64 |     storage.write_value(key1, 40)
 65 | 
 66 |     assert storage.get_value(key1) == 40.0
 67 | 
 68 |     storage.clear()
 69 |     assert len(storage) == 0
 70 | 
 71 |     storage = LocalMemoryStorage()
 72 | 
 73 |     for k, v in DATA:
 74 |         storage.write_value(k, v)
 75 | 
 76 |     assert len(storage) == len(DATA)
 77 | 
 78 |     items = list(storage.items())
 79 |     assert len(items) == 4
 80 | 
 81 |     for name, labels in items:
 82 | 
 83 |         if name == "metric_counter_name":
 84 |             for label, label_data in labels:
 85 |                 assert len(label_data) == 1
 86 |         if name == "metric_gauge_name":
 87 |             for label, label_data in labels:
 88 |                 assert len(label_data) == 1
 89 | 
 90 |         if name == "metric_histogram_name":
 91 |             for label, label_data in labels:
 92 |                 assert len(label_data) == 6
 93 | 
 94 |         if name == "metric_summary_name":
 95 |             for label, label_data in labels:
 96 |                 assert len(label_data) == 2
 97 | 
 98 |         assert len(labels) == 2
 99 | 
100 |     assert len(items) == 4
101 | 
102 | 
103 | def test_local_storage_threading(measure_time, iterations, num_workers):
104 |     storage = LocalMemoryStorage()
105 | 
106 |     ITERATIONS = iterations
107 | 
108 |     with measure_time("threading writes") as mt:
109 |         def f1():
110 |             for _ in xrange(ITERATIONS):
111 |                 for x in DATA:
112 |                     storage.inc_value(x[0], x[1])
113 | 
114 |         def f2():
115 |             for _ in xrange(ITERATIONS):
116 |                 for x in DATA:
117 |                     storage.inc_value(x[0], x[1])
118 | 
119 |         def f3():
120 |             for _ in xrange(ITERATIONS):
121 |                 for x in DATA:
122 |                     storage.inc_value(x[0], x[1])
123 | 
124 |         workers = []
125 |         for _ in xrange(num_workers):
126 |             func = random.choice([f1, f2, f3])
127 |             t = threading.Thread(target=func)
128 | 
129 |             t.start()
130 |             workers.append(t)
131 | 
132 |         for x in workers:
133 |             x.join()
134 | 
135 |         mt.set_num_ops(ITERATIONS * len(workers) * len(DATA))
136 | 
137 |     with measure_time("threading reads") as mt:
138 |         mt.set_num_ops(len(DATA))
139 | 
140 |         for x in DATA:
141 |             assert storage.get_value(x[0]) == x[1] * ITERATIONS * len(workers)
142 | 


--------------------------------------------------------------------------------
/tests/test_uwsgi_collector.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | import os
  4 | import random
  5 | from multiprocessing import Process
  6 | 
  7 | import uwsgi
  8 | from pyprometheus.contrib.uwsgi_features import UWSGICollector, UWSGIStorage, UWSGIFlushStorage
  9 | from pyprometheus.registry import BaseRegistry
 10 | from pyprometheus.utils.exposition import registry_to_text
 11 | try:
 12 |     xrange = xrange
 13 | except Exception:
 14 |     xrange = range
 15 | 
 16 | 
 17 | def test_uwsgi_collector():
 18 |     registry = BaseRegistry()
 19 |     uwsgi_collector = UWSGICollector(namespace="uwsgi_namespace", labels={"env_role": "test"})
 20 | 
 21 |     registry.register(uwsgi_collector)
 22 | 
 23 |     collectors = {x.name: x for x in registry.collect()}
 24 | 
 25 |     metrics_count = sorted(map(lambda x: x.split(" ")[2],
 26 |                            filter(lambda x: x.startswith("# HELP"), [x for x in registry_to_text(registry).split("\n")])))
 27 | 
 28 |     assert len(metrics_count) == len(set(metrics_count))
 29 | 
 30 |     assert len(registry_to_text(registry).split("\n")) == 60
 31 | 
 32 |     assert collectors["uwsgi_namespace:buffer_size_bytes"].get_samples()[0].value == uwsgi.buffer_size
 33 |     assert collectors["uwsgi_namespace:processes_total"].get_samples()[0].value == uwsgi.numproc
 34 |     assert collectors["uwsgi_namespace:requests_total"].get_samples()[0].value == uwsgi.total_requests()
 35 | 
 36 |     for name in ["requests", "respawn_count", "running_time", "exceptions", "delta_requests"]:
 37 |         assert collectors["uwsgi_namespace:process:{0}".format(name)].get_samples()[0].value == uwsgi.workers()[0][name]
 38 | 
 39 |     assert uwsgi_collector.metric_name("test") == "uwsgi_namespace:test"
 40 | 
 41 | 
 42 | DATA = (
 43 |     ((2, "metric_gauge_name", "", (("label1", "value1"), ("label2", "value2"))), 5),
 44 |     ((3, "metric_counter_name", "", (("label1", "value1"), ("label2", "value2"))), 7),
 45 |     ((5, "metric_summary_name", "_sum", (("label1", "value1"), ("label2", "value2"))), 4),
 46 |     ((7, "metric_summary_name", "_count", (("label1", "value1"), ("label2", "value2"))), 1),
 47 |     ((11, "metric_histogram_name", "_sum", (("label1", "value1"), ("label2", "value2"))), 6),
 48 |     ((12, "metric_histogram_name", "_count", (("label1", "value1"), ("label2", "value2"))), 1),
 49 |     ((13, "metric_histogram_name", "_bucket", (("bucket", "0.005"), ("label1", "value1"), ("label2", "value2"))), 0),
 50 |     ((13, "metric_histogram_name", "_bucket", (("bucket", "0.01"), ("label1", "value1"), ("label2", "value2"))), 0),
 51 |     ((13, "metric_histogram_name", "_bucket", (("bucket", "7.5"), ("label1", "value1"), ("label2", "value2"))), 1),
 52 |     ((13, "metric_histogram_name", "_bucket", (("bucket", "+Inf"), ("label1", "value1"), ("label2", "value2"))), 1),
 53 |     ((2, "metric_gauge_name", "", (("label1", "value3"), ("label2", "value4"))), 5),
 54 |     ((3, "metric_counter_name", "", (("label1", "value3"), ("label2", "value4"))), 7),
 55 |     ((5, "metric_summary_name", "_sum", (("label1", "value3"), ("label2", "value4"))), 4),
 56 |     ((7, "metric_summary_name", "_count", (("label1", "value3"), ("label2", "value4"))), 1),
 57 |     ((11, "metric_histogram_name", "_sum", (("label1", "value3"), ("label2", "value4"))), 6),
 58 |     ((12, "metric_histogram_name", "_count", (("label1", "value3"), ("label2", "value4"))), 1),
 59 |     ((13, "metric_histogram_name", "_bucket", (("bucket", "0.005"), ("label1", "value3"), ("label2", "value4"))), 0),
 60 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 0.01), ("label1", "value3"), ("label2", "value4"))), 0),
 61 |     ((13, "metric_histogram_name", "_bucket", (("bucket", 7.5), ("label1", "value3"), ("label2", "value4"))), 1),
 62 |     ((13, "metric_histogram_name", "_bucket", (("bucket", float("inf")), ("label1", "value3"), ("label2", "value4"))), 1))
 63 | 
 64 | 
 65 | def test_uwsgi_storage():
 66 | 
 67 |     storage = UWSGIStorage(0)
 68 |     storage2 = UWSGIStorage(0)
 69 | 
 70 |     # 100 pages
 71 |     assert len(storage.m) == 409600 == 100 * 1024 * 4
 72 | 
 73 |     assert (storage.get_area_size()) == 14
 74 | 
 75 |     assert storage.m[15] == "\x00"
 76 | 
 77 |     with storage.lock():
 78 | 
 79 |         assert storage.wlocked
 80 |         assert storage.rlocked
 81 | 
 82 |     assert not storage.wlocked
 83 |     assert not storage.rlocked
 84 | 
 85 |     with storage.rlock():
 86 |         assert not storage.wlocked
 87 |         assert storage.rlocked
 88 | 
 89 |     assert not storage.wlocked
 90 |     assert not storage.rlocked
 91 | 
 92 |     assert storage.is_actual
 93 | 
 94 |     area_sign = storage.get_area_sign()
 95 | 
 96 |     assert area_sign == storage2.get_area_sign()
 97 | 
 98 |     storage.m[storage.SIGN_POSITION + 2] = os.urandom(1)
 99 | 
100 |     assert not storage.is_actual
101 | 
102 |     s = "keyname"
103 |     assert storage.get_string_padding(s) == 5
104 | 
105 |     assert len(s.encode("utf-8")) + storage.get_string_padding(s) == 12
106 | 
107 |     assert storage.validate_actuality()
108 | 
109 |     assert storage.is_actual
110 | 
111 |     assert storage.get_key_position("keyname") == ([14, 18, 25, 33], True)
112 | 
113 |     assert (storage.get_area_size()) == 33
114 | 
115 |     assert storage.get_key_size("keyname") == 24
116 | 
117 |     storage.write_value("keyname", 10)
118 | 
119 |     assert storage.get_value("keyname") == 10.0
120 | 
121 |     storage.clear()
122 | 
123 |     assert storage.get_area_size() == 0 == storage2.get_area_size()
124 | 
125 |     storage.validate_actuality()
126 | 
127 |     assert storage.get_area_size() == 14 == storage2.get_area_size()
128 | 
129 |     storage.write_value(DATA[0][0], DATA[0][1])
130 | 
131 |     assert storage.get_key_size(DATA[0][0]) == 108
132 | 
133 |     assert storage.get_area_size() == 122 == storage2.get_area_size()
134 | 
135 |     assert storage2.get_value(DATA[0][0]) == DATA[0][1] == storage.get_value(DATA[0][0])
136 | 
137 |     for x in DATA:
138 |         storage.write_value(x[0], x[1])
139 |         assert storage.get_value(x[0]) == x[1]
140 | 
141 |     for x in DATA:
142 |         assert storage.get_value(x[0]) == x[1]
143 | 
144 |     assert len(storage) == len(DATA) == 20
145 | 
146 |     assert storage.get_area_size() == 2531
147 |     assert not storage2.is_actual
148 | 
149 |     assert storage.is_actual
150 | 
151 |     storage2.validate_actuality()
152 | 
153 |     for x in DATA:
154 |         assert storage2.get_value(x[0]) == x[1]
155 | 
156 | 
157 | def test_multiprocessing(measure_time, iterations, num_workers):
158 | 
159 |     storage = UWSGIStorage(0)
160 |     storage2 = UWSGIStorage(0)
161 |     storage3 = UWSGIStorage(0)
162 |     ITERATIONS = iterations
163 | 
164 |     with measure_time("multiprocessing writes {0}".format(ITERATIONS)) as mt:
165 |         def f1():
166 |             for _ in xrange(ITERATIONS):
167 |                 for x in DATA:
168 |                     storage.inc_value(x[0], x[1])
169 | 
170 |         def f2():
171 |             for _ in xrange(ITERATIONS):
172 |                 for x in DATA:
173 |                     storage2.inc_value(x[0], x[1])
174 | 
175 |         def f3():
176 |             for _ in xrange(ITERATIONS):
177 |                 for x in DATA:
178 |                     storage3.inc_value(x[0], x[1])
179 | 
180 |         workers = []
181 |         for _ in xrange(num_workers):
182 |             func = random.choice([f1, f2, f3])
183 |             p = Process(target=func)
184 |             p.start()
185 |             workers.append(p)
186 | 
187 |         for x in workers:
188 |             x.join()
189 | 
190 |         mt.set_num_ops(ITERATIONS * len(workers) * len(DATA))
191 | 
192 |     with measure_time("multiprocessing reads") as mt:
193 |         mt.set_num_ops(3 * len(DATA))
194 | 
195 |         for x in DATA:
196 |             assert storage2.get_value(x[0]) == storage.get_value(x[0]) == storage3.get_value(x[0]) == x[1] * ITERATIONS * len(workers)
197 | 
198 | 
199 | def test_uwsgi_flush_storage():
200 | 
201 |     storage1 = UWSGIFlushStorage(0)
202 |     storage2 = UWSGIFlushStorage(0)
203 | 
204 |     for x in xrange(10):
205 |         for k, v in DATA:
206 |             storage1.inc_value(k, v)
207 | 
208 |             storage1.get_value(k) == v
209 | 
210 |             storage2.get_value(k) == 0
211 | 
212 |         storage1.flush()
213 | 
214 |     for x in DATA:
215 |         storage1.get_value(x[0]) == 0
216 |         storage1.persistent_storage.get_value(x[0]) == x[1] * 10
217 | 
218 | 
219 | def test_uwsgi_flush_storage_multiprocessing(measure_time, iterations, num_workers):
220 |     storage = UWSGIFlushStorage(0)
221 |     storage2 = UWSGIFlushStorage(0)
222 |     storage3 = UWSGIFlushStorage(0)
223 |     ITERATIONS = iterations
224 |     with measure_time("flush storage multiprocessing writes {0}".format(ITERATIONS)) as mt:
225 |         def f1():
226 |             for _ in xrange(ITERATIONS):
227 |                 for x in DATA:
228 |                     storage.inc_value(x[0], x[1])
229 | 
230 |                 storage.flush()
231 | 
232 |         def f2():
233 |             for _ in xrange(ITERATIONS):
234 |                 for x in DATA:
235 |                     storage2.inc_value(x[0], x[1])
236 | 
237 |                 storage2.flush()
238 | 
239 |         def f3():
240 |             for _ in xrange(ITERATIONS):
241 |                 for x in DATA:
242 |                     storage3.inc_value(x[0], x[1])
243 | 
244 |                 storage3.flush()
245 | 
246 |         workers = []
247 |         for _ in xrange(num_workers):
248 |             func = random.choice([f1, f2, f3])
249 |             p = Process(target=func)
250 |             p.start()
251 |             workers.append(p)
252 | 
253 |         for x in workers:
254 |             x.join()
255 | 
256 |         mt.set_num_ops(ITERATIONS * len(workers) * len(DATA))
257 | 
258 |     storage.flush()
259 |     storage2.flush()
260 |     storage3.flush()
261 | 
262 |     with measure_time("flush storage multiprocessing reads") as mt:
263 |         mt.set_num_ops(3 * len(DATA))
264 | 
265 |         for x in DATA:
266 |             assert storage2.get_value(x[0]) == storage.get_value(x[0]) == storage3.get_value(x[0]) == 0
267 |             assert storage2.persistent_storage.get_value(x[0]) == storage.persistent_storage.get_value(x[0]) == storage3.persistent_storage.get_value(x[0])
268 | 
269 |             assert storage.persistent_storage.get_value(x[0]) == x[1] * ITERATIONS * len(workers)
270 | 
271 | 
272 | def test_uwsgi_storage_metrics(iterations):
273 |     registry = BaseRegistry()
274 | 
275 |     storage = UWSGIStorage(0, namespace="namespace", stats=True)
276 | 
277 |     registry.register(storage)
278 | 
279 |     for x in xrange(iterations):
280 |         for k, v in DATA:
281 |             storage.inc_value(k, v)
282 | 
283 |     collectors = {x.name: x for x in registry.collect()}
284 | 
285 |     metric = collectors["namespace:memory_size"]
286 |     assert metric.get_samples()[0].value == storage.get_area_size()
287 | 
288 |     metric = collectors["namespace:num_keys"]
289 |     assert metric.get_samples()[0].value == 20
290 | 


--------------------------------------------------------------------------------
/tests_requirements.txt:
--------------------------------------------------------------------------------
 1 | flake8==3.2.1
 2 | tox==2.3.2
 3 | #tox-pyenv==2.3.2
 4 | 
 5 | 
 6 | ipdb
 7 | uwsgi==2.0.14
 8 | pytest==3.0.6
 9 | pytest-cov==2.4.0
10 | pytest-flake8==0.8.1
11 | flake8-quotes==0.9.0
12 | flake8-comprehensions==1.2.1
13 | 


--------------------------------------------------------------------------------
/tools/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:2.7.8
 2 | 
 3 | ENV PYTHONUNBUFFERED=1
 4 | 
 5 | WORKDIR /usr/src/app
 6 | 
 7 | RUN apt-get update && apt-get install -y \
 8 |     libxml2-dev libxslt-dev python-dev  \
 9 |     libyaml-dev \
10 |     graphviz
11 | 
12 | 
13 | COPY tests_requirements.txt /tests_requirements.txt
14 | 
15 | # Install pyenv for tox
16 | RUN git clone https://github.com/yyuu/pyenv.git ~/.pyenv
17 | ENV HOME /root
18 | ENV PYENV_ROOT $HOME/.pyenv
19 | ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
20 | 
21 | RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bash_profile
22 | RUN echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bash_profile
23 | RUN echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
24 | 
25 | RUN exec $SHELL
26 | 
27 | RUN pyenv install 2.7 && \
28 |     pyenv install 3.2  && \
29 |     pyenv install 3.3.0 && \
30 |     pyenv install 3.4.0 && \
31 |     pyenv install 3.5.0 && \
32 |     pyenv install 3.6.0 && \
33 |     pyenv install 3.7-dev && \
34 |     pyenv install pypy-4.0.1
35 | 
36 | # RUN pyenv local 2.7 3.2 3.3.0 3.4.0 3.5.0 3.6.0 3.7-dev pypy-4.0.1
37 | 
38 | RUN pip install -U pip setuptools wheel==0.30.0a0 pyparsing==2.1.10 twine
39 | RUN pip install -U plop gprof2dot ipython
40 | 
41 | RUN pip install -U -r /tests_requirements.txt
42 | 
43 | 
44 | ENTRYPOINT ["/bin/bash", "-c"]
45 | 


--------------------------------------------------------------------------------
/tools/Makefile:
--------------------------------------------------------------------------------
 1 | clean-containers:
 2 | 	docker ps -q -f status=exited | xargs docker rm
 3 | 
 4 | build-images:
 5 | 	@echo "Build docker images"
 6 | 	docker-compose -f tools/compose-config.yml build # --force-rm
 7 | 
 8 | 
 9 | run-cmd:
10 | 	@echo "Execute command in docker"
11 | 	docker-compose -f tools/compose-config.yml run dev_pyprometheus $(DOCKER_CMD)
12 | 
13 | shell:
14 | 	@echo "Running shell in docker"
15 | 	docker-compose -f tools/compose-config.yml run dev_pyprometheus /bin/bash
16 | 


--------------------------------------------------------------------------------
/tools/compose-config.yml:
--------------------------------------------------------------------------------
 1 | version: "2.0"
 2 | 
 3 | services:
 4 |   dev_pyprometheus:
 5 |     image: pypropetheus_dev:latest
 6 |     build:
 7 |       dockerfile: tools/Dockerfile
 8 |       context: ..
 9 |     environment:
10 |       - PYTHONUNBUFFERED=1
11 |     command:
12 |       - "python --version"
13 |     volumes:
14 |       - ../:/usr/src/app/
15 |       - ~/.pypirc:/root/.pypirc
16 | 
17 |     networks:
18 |       - default
19 | 
20 | networks:
21 |   default:


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
 1 | # Tox (http://codespeak.net/~hpk/tox/) is a tool for running tests
 2 | # in multiple virtualenvs. This configuration file will run the
 3 | # test suite on all supported python versions. To use it, "pip install tox"
 4 | # and then run "tox" from this directory.
 5 | 
 6 | [tox]
 7 | envlist = py27#, py33, py34, py35
 8 | 
 9 | [testenv]
10 | deps = -rtests_requirements.txt
11 | 
12 | commands =
13 |     pyenv local 2.7 3.2 3.3.0 3.4.0 3.5.0 3.6.0 3.7-dev pypy-4.0.1
14 |     #pip install -e .[tests]
15 |     uwsgi --pyrun setup.py --pyargv test --sharedarea=100 --enable-threads


--------------------------------------------------------------------------------