├── .gitignore ├── .gitreview ├── .pre-commit-config.yaml ├── .stestr.conf ├── .zuul.yaml ├── CONTRIBUTING.rst ├── LICENSE ├── README.rst ├── bindep.txt ├── devstack ├── README.rst ├── lib │ └── osprofiler ├── plugin.sh └── settings ├── doc ├── requirements.txt ├── source │ ├── Makefile │ ├── conf.py │ ├── index.rst │ └── user │ │ ├── api.rst │ │ ├── background.rst │ │ ├── collectors.rst │ │ ├── history.rst │ │ ├── index.rst │ │ ├── integration.rst │ │ └── similar_projects.rst └── specs │ ├── README.rst │ ├── implemented │ ├── README.rst │ ├── make_paste_ini_config_optional.rst │ └── multi_backend_support.rst │ ├── in-progress │ ├── README.rst │ ├── better_devstack_integration.rst │ └── integration_testing.rst │ └── template.rst ├── osprofiler ├── __init__.py ├── _utils.py ├── cmd │ ├── __init__.py │ ├── cliutils.py │ ├── commands.py │ ├── shell.py │ └── template.html ├── drivers │ ├── __init__.py │ ├── base.py │ ├── elasticsearch_driver.py │ ├── jaeger.py │ ├── loginsight.py │ ├── messaging.py │ ├── mongodb.py │ ├── otlp.py │ ├── redis_driver.py │ └── sqlalchemy_driver.py ├── exc.py ├── hacking │ ├── __init__.py │ └── checks.py ├── initializer.py ├── notifier.py ├── opts.py ├── profiler.py ├── requests.py ├── sqlalchemy.py ├── tests │ ├── __init__.py │ ├── functional │ │ ├── __init__.py │ │ ├── config.cfg │ │ └── test_driver.py │ ├── test.py │ └── unit │ │ ├── __init__.py │ │ ├── cmd │ │ ├── __init__.py │ │ └── test_shell.py │ │ ├── doc │ │ ├── __init__.py │ │ └── test_specs.py │ │ ├── drivers │ │ ├── __init__.py │ │ ├── test_base.py │ │ ├── test_elasticsearch.py │ │ ├── test_loginsight.py │ │ ├── test_messaging.py │ │ ├── test_mongodb.py │ │ ├── test_otlp.py │ │ └── test_redis_driver.py │ │ ├── test_initializer.py │ │ ├── test_notifier.py │ │ ├── test_opts.py │ │ ├── test_profiler.py │ │ ├── test_sqlalchemy.py │ │ ├── test_utils.py │ │ └── test_web.py └── web.py ├── playbooks └── osprofiler-post.yaml ├── pyproject.toml ├── releasenotes ├── notes │ ├── add-reno-996dd44974d53238.yaml │ ├── add-requests-profiling-761e09f243d36966.yaml │ ├── drop-jaeger-container-when-unstacking-e8fcdc036f80158a.yaml │ ├── drop-python-2-7-73d3113c69d724d6.yaml │ ├── jaeger-add-process-tags-79d5f5d7a0b049ef.yaml │ ├── jaeger-service-name-prefix-72878a930f700878.yaml │ ├── otlp-driver-cb932038ad580ac2.yaml │ ├── redis-improvement-d4c91683fc89f570.yaml │ ├── remove-py38-e2c2723282ebbf9f.yaml │ ├── remove-strict-redis-9eb43d30c9c1fc43.yaml │ └── retire-jaeger-driver-d8add44c5522ad7a.yaml └── source │ ├── _static │ └── .placeholder │ ├── _templates │ └── .placeholder │ ├── conf.py │ ├── index.rst │ ├── ocata.rst │ ├── pike.rst │ ├── queens.rst │ ├── rocky.rst │ ├── stein.rst │ ├── train.rst │ ├── unreleased.rst │ ├── ussuri.rst │ └── victoria.rst ├── requirements.txt ├── setup.cfg ├── setup.py ├── test-requirements.txt ├── tools ├── lint.py └── patch_tox_venv.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | AUTHORS 2 | ChangeLog 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg* 10 | dist 11 | build 12 | _build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | cover 30 | .stestr/ 31 | *.sqlite 32 | .venv 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mr Developer 38 | .mr.developer.cfg 39 | .project 40 | .pydevproject 41 | .idea 42 | 43 | # Docs generated 44 | doc/source/contributor/modules 45 | 46 | # reno build 47 | releasenotes/build 48 | RELEASENOTES.rst 49 | releasenotes/notes/reno.cache 50 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.opendev.org 3 | port=29418 4 | project=openstack/osprofiler.git 5 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | # Replaces or checks mixed line ending 7 | - id: mixed-line-ending 8 | args: ['--fix', 'lf'] 9 | exclude: '.*\.(svg)$' 10 | # Forbid files which have a UTF-8 byte-order marker 11 | - id: check-byte-order-marker 12 | # Checks that non-binary executables have a proper shebang 13 | - id: check-executables-have-shebangs 14 | # Check for files that contain merge conflict strings. 15 | - id: check-merge-conflict 16 | # Check for debugger imports and py37+ breakpoint() 17 | # calls in python source 18 | - id: debug-statements 19 | - id: check-yaml 20 | files: .*\.(yaml|yml)$ 21 | - repo: https://opendev.org/openstack/hacking 22 | rev: 7.0.0 23 | hooks: 24 | - id: hacking 25 | additional_dependencies: [] 26 | - repo: https://github.com/PyCQA/bandit 27 | rev: 1.7.10 28 | hooks: 29 | - id: bandit 30 | - repo: https://github.com/asottile/pyupgrade 31 | rev: v3.18.0 32 | hooks: 33 | - id: pyupgrade 34 | args: [--py3-only] 35 | -------------------------------------------------------------------------------- /.stestr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=${OS_TEST_PATH:-./osprofiler/tests/unit} 3 | top_dir=./ 4 | 5 | -------------------------------------------------------------------------------- /.zuul.yaml: -------------------------------------------------------------------------------- 1 | - project: 2 | templates: 3 | - check-requirements 4 | - lib-forward-testing-python3 5 | - openstack-cover-jobs 6 | - openstack-python3-jobs 7 | - periodic-stable-jobs 8 | - publish-openstack-docs-pti 9 | - release-notes-jobs-python3 10 | check: 11 | jobs: 12 | - openstack-tox-functional-py39 13 | - openstack-tox-functional-py312 14 | - tempest-smoke-py3-osprofiler-redis 15 | - tempest-smoke-py3-osprofiler-sqlalchemy 16 | gate: 17 | jobs: 18 | - openstack-tox-functional-py39 19 | 20 | - job: 21 | name: tempest-smoke-py3-osprofiler-redis 22 | parent: tempest-full-py3 23 | voting: false 24 | post-run: playbooks/osprofiler-post.yaml 25 | description: | 26 | Run full tempest on py3 with profiling enabled (redis driver) 27 | required-projects: 28 | - openstack/osprofiler 29 | vars: 30 | tox_envlist: smoke 31 | devstack_localrc: 32 | OSPROFILER_COLLECTOR: redis 33 | OSPROFILER_HMAC_KEYS: SECRET_KEY 34 | devstack_plugins: 35 | osprofiler: https://opendev.org/openstack/osprofiler 36 | 37 | - job: 38 | name: tempest-smoke-py3-osprofiler-sqlalchemy 39 | parent: tempest-full-py3 40 | voting: false 41 | post-run: playbooks/osprofiler-post.yaml 42 | description: | 43 | Run full tempest on py3 with profiling enabled (sqlalchemy driver) 44 | required-projects: 45 | - openstack/osprofiler 46 | vars: 47 | tox_envlist: smoke 48 | devstack_localrc: 49 | OSPROFILER_COLLECTOR: sqlalchemy 50 | OSPROFILER_HMAC_KEYS: SECRET_KEY 51 | devstack_plugins: 52 | osprofiler: https://opendev.org/openstack/osprofiler 53 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | If you would like to contribute to the development of OpenStack, 2 | you must follow the steps in this page: 3 | 4 | https://docs.openstack.org/infra/manual/developers.html 5 | 6 | Once those steps have been completed, changes to OpenStack 7 | should be submitted for review via the Gerrit tool, following 8 | the workflow documented at: 9 | 10 | https://docs.openstack.org/infra/manual/developers.html#development-workflow 11 | 12 | Pull requests submitted through GitHub will be ignored. 13 | 14 | Bugs should be filed on Launchpad, not GitHub: 15 | 16 | https://bugs.launchpad.net/osprofiler 17 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =================================================== 2 | OSProfiler -- Library for cross-project profiling 3 | =================================================== 4 | 5 | .. image:: https://governance.openstack.org/tc/badges/osprofiler.svg 6 | :target: https://governance.openstack.org/tc/reference/tags/index.html 7 | 8 | .. Change things from this point on 9 | 10 | .. image:: https://img.shields.io/pypi/v/osprofiler.svg 11 | :target: https://pypi.org/project/osprofiler/ 12 | :alt: Latest Version 13 | 14 | .. image:: https://img.shields.io/pypi/dm/osprofiler.svg 15 | :target: https://pypi.org/project/osprofiler/ 16 | :alt: Downloads 17 | 18 | OSProfiler provides a tiny but powerful library that is used by 19 | most (soon to be all) OpenStack projects and their python clients. It 20 | provides functionality to be able to generate 1 trace per request, that goes 21 | through all involved services. This trace can then be extracted and used 22 | to build a tree of calls which can be quite handy for a variety of 23 | reasons (for example in isolating cross-project performance issues). 24 | 25 | * Free software: Apache license 26 | * Documentation: https://docs.openstack.org/osprofiler/latest/ 27 | * Source: https://opendev.org/openstack/osprofiler 28 | * Bugs: https://bugs.launchpad.net/osprofiler 29 | * Release notes: https://docs.openstack.org/releasenotes/osprofiler 30 | -------------------------------------------------------------------------------- /bindep.txt: -------------------------------------------------------------------------------- 1 | rabbitmq-server [test] 2 | redis [test platform:rpm] 3 | redis-server [test platform:dpkg] 4 | -------------------------------------------------------------------------------- /devstack/README.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | Enabling OSProfiler using DevStack 3 | ================================== 4 | 5 | This directory contains the files necessary to run OpenStack with enabled 6 | OSProfiler in DevStack. 7 | 8 | OSProfiler can send trace data into different collectors. There are 2 parameters 9 | that control this: 10 | 11 | * ``OSPROFILER_COLLECTOR`` specifies which collector to install in DevStack. 12 | By default OSProfiler plugin does not install anything, thus default 13 | messaging driver will be used. 14 | 15 | Possible values: 16 | 17 | * ```` - default messaging driver is used 18 | * ``redis`` - Redis is installed 19 | * ``jaeger`` - Jaeger is installed 20 | * ``sqlalchemy`` - SQLAlchemy driver is installed 21 | 22 | The default value of ``OSPROFILER_CONNECTION_STRING`` is set automatically 23 | depending on ``OSPROFILER_COLLECTOR`` value. 24 | 25 | * ``OSPROFILER_CONNECTION_STRING`` specifies which driver is used by OSProfiler. 26 | 27 | Possible values: 28 | 29 | * ``messaging://`` - use messaging as trace collector (with the transport configured by oslo.messaging) 30 | * ``redis://[:password]@host[:port][/db]`` - use Redis as trace storage 31 | * ``elasticsearch://host:port`` - use Elasticsearch as trace storage 32 | * ``mongodb://host:port`` - use MongoDB as trace storage 33 | * ``loginsight://username:password@host`` - use LogInsight as trace collector/storage 34 | * ``jaeger://host:port`` - use Jaeger as trace collector 35 | * ``mysql+pymysql://username:password@host/profiler?charset=utf8`` - use SQLAlchemy driver with MySQL database 36 | 37 | 38 | To configure DevStack and enable OSProfiler edit ``${DEVSTACK_DIR}/local.conf`` 39 | file and add the following to ``[[local|localrc]]`` section: 40 | 41 | * to use Redis collector:: 42 | 43 | enable_plugin osprofiler https://opendev.org/openstack/osprofiler master 44 | OSPROFILER_COLLECTOR=redis 45 | 46 | OSProfiler plugin will install Redis and configure OSProfiler to use Redis driver 47 | 48 | * to use specified driver:: 49 | 50 | enable_plugin osprofiler https://opendev.org/openstack/osprofiler master 51 | OSPROFILER_CONNECTION_STRING= 52 | 53 | the driver is chosen depending on the value of 54 | ``OSPROFILER_CONNECTION_STRING`` variable (refer to the next section for 55 | details) 56 | 57 | 58 | Run DevStack as normal:: 59 | 60 | $ ./stack.sh 61 | 62 | 63 | Config variables 64 | ---------------- 65 | 66 | **OSPROFILER_HMAC_KEYS** - a set of HMAC secrets, that are used for triggering 67 | of profiling in OpenStack services: only the requests that specify one of these 68 | keys in HTTP headers will be profiled. E.g. multiple secrets are specified as 69 | a comma-separated list of string values:: 70 | 71 | OSPROFILER_HMAC_KEYS=swordfish,foxtrot,charlie 72 | 73 | **OSPROFILER_CONNECTION_STRING** - connection string to identify the driver. 74 | Default value is ``messaging://`` refers to messaging driver. For a full 75 | list of drivers please refer to 76 | ``https://opendev.org/openstack/osprofiler/src/branch/master/osprofiler/drivers``. 77 | Example: enable ElasticSearch driver with the server running on localhost:: 78 | 79 | OSPROFILER_CONNECTION_STRING=elasticsearch://127.0.0.1:9200 80 | 81 | **OSPROFILER_COLLECTOR** - controls which collector to install into DevStack. 82 | The driver is then chosen automatically based on the collector. Empty value assumes 83 | that the default messaging driver is used. 84 | Example: enable Redis collector:: 85 | 86 | OSPROFILER_COLLECTOR=redis 87 | 88 | **OSPROFILER_TRACE_SQLALCHEMY** - controls tracing of SQL statements. If enabled, 89 | all SQL statements processed by SQL Alchemy are added into traces. By default enabled. 90 | Example: disable SQL statements tracing:: 91 | 92 | OSPROFILER_TRACE_SQLALCHEMY=False 93 | -------------------------------------------------------------------------------- /devstack/lib/osprofiler: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # lib/osprofiler 4 | # Functions to control the configuration and operation of the **OSProfiler** 5 | 6 | # Save trace setting 7 | XTRACE=$(set +o | grep xtrace) 8 | set +o xtrace 9 | 10 | 11 | # Defaults 12 | # -------- 13 | 14 | CONF_FILES=( 15 | $CINDER_CONF 16 | $HEAT_CONF 17 | $KEYSTONE_CONF 18 | $NOVA_CONF 19 | $NEUTRON_CONF 20 | $GLANCE_API_CONF 21 | $GLANCE_REGISTRY_CONF 22 | $TROVE_CONF 23 | $TROVE_CONDUCTOR_CONF 24 | $TROVE_GUESTAGENT_CONF 25 | $TROVE_TASKMANAGER_CONF 26 | $SENLIN_CONF 27 | $MAGNUM_CONF 28 | $MANILA_CONF 29 | $ZUN_CONF 30 | $PLACEMENT_CONF 31 | ) 32 | 33 | # Add config files of Nova Cells 34 | NOVA_NUM_CELLS=${NOVA_NUM_CELLS:-1} 35 | for i in $(seq 1 ${NOVA_NUM_CELLS}); do 36 | # call function `conductor_conf` defined in lib/nova to get file name 37 | conf=$(conductor_conf $i) 38 | CONF_FILES+=(${conf}) 39 | done 40 | 41 | 42 | # Functions 43 | # --------- 44 | 45 | function install_redis() { 46 | if is_fedora; then 47 | install_package redis 48 | elif is_ubuntu; then 49 | install_package redis-server 50 | elif is_suse; then 51 | install_package redis 52 | else 53 | exit_distro_not_supported "redis installation" 54 | fi 55 | 56 | start_service redis 57 | 58 | pip_install_gr redis 59 | } 60 | 61 | function install_jaeger_backend() { 62 | if is_ubuntu; then 63 | install_package docker.io 64 | start_service docker 65 | add_user_to_group $STACK_USER docker 66 | sg docker -c "docker run -d --name jaeger -e COLLECTOR_OTLP_ENABLED=true -p 6831:6831/udp -p 16686:16686 -p 4318:4318 jaegertracing/all-in-one:1.42" 67 | else 68 | exit_distro_not_supported "docker.io installation" 69 | fi 70 | } 71 | 72 | function install_jaeger() { 73 | install_jaeger_backend 74 | pip_install jaeger-client 75 | } 76 | 77 | function install_otlp() { 78 | # For OTLP we use Jaeger backend but any OTLP compatible backend 79 | # can be used. 80 | install_jaeger_backend 81 | pip_install opentelemetry-sdk opentelemetry-exporter-otlp 82 | } 83 | 84 | function drop_jaeger() { 85 | sg docker -c 'docker rm jaeger --force' 86 | } 87 | 88 | function install_elasticsearch() { 89 | if is_ubuntu; then 90 | install_package docker.io 91 | start_service docker 92 | add_user_to_group $STACK_USER docker 93 | # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html#docker-cli-run-dev-mode 94 | sg docker -c 'docker run -d --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:5.6.14' 95 | else 96 | exit_distro_not_supported "docker.io installation" 97 | fi 98 | 99 | pip_install elasticsearch 100 | } 101 | 102 | function install_mongodb { 103 | pip_install pymongo 104 | if is_ubuntu; then 105 | install_package mongodb-server 106 | start_service mongodb 107 | elif is_fedora; then 108 | install_package mongodb 109 | install_package mongodb-server 110 | start_service mongod 111 | else 112 | exit_distro_not_supported "mongodb installation" 113 | fi 114 | } 115 | 116 | function install_osprofiler_collector() { 117 | if [ -z "$OSPROFILER_COLLECTOR" ]; then 118 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"messaging://"} 119 | elif [ "$OSPROFILER_COLLECTOR" == "redis" ]; then 120 | install_redis 121 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"redis://localhost:6379"} 122 | elif [ "$OSPROFILER_COLLECTOR" == "jaeger" ]; then 123 | install_jaeger 124 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"jaeger://localhost:6831"} 125 | elif [ "$OSPROFILER_COLLECTOR" == "otlp" ]; then 126 | install_otlp 127 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"otlp://localhost:4318"} 128 | elif [ "$OSPROFILER_COLLECTOR" == "elasticsearch" ]; then 129 | install_elasticsearch 130 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"elasticsearch://elastic:changeme@localhost:9200"} 131 | elif [ "$OSPROFILER_COLLECTOR" == "mongodb" ]; then 132 | install_mongodb 133 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"mongodb://localhost:27017"} 134 | elif [ "$OSPROFILER_COLLECTOR" == "sqlalchemy" ]; then 135 | local db=`database_connection_url osprofiler` 136 | OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-${db}} 137 | recreate_database osprofiler 138 | else 139 | die $LINENO "OSProfiler collector $OSPROFILER_COLLECTOR is not supported" 140 | fi 141 | 142 | echo ${OSPROFILER_CONNECTION_STRING} > $HOME/.osprofiler_connection_string 143 | } 144 | 145 | function configure_osprofiler() { 146 | 147 | for conf in ${CONF_FILES[@]}; do 148 | if [ -f $conf ] 149 | then 150 | iniset $conf profiler enabled True 151 | iniset $conf profiler trace_sqlalchemy $OSPROFILER_TRACE_SQLALCHEMY 152 | iniset $conf profiler hmac_keys $OSPROFILER_HMAC_KEYS 153 | iniset $conf profiler connection_string $OSPROFILER_CONNECTION_STRING 154 | fi 155 | done 156 | 157 | # Keystone is already running, should be reloaded to apply osprofiler config 158 | reload_service devstack@keystone 159 | } 160 | 161 | function configure_osprofiler_in_tempest() { 162 | 163 | iniset $TEMPEST_CONFIG profiler key $OSPROFILER_HMAC_KEYS 164 | } 165 | 166 | 167 | # Restore xtrace 168 | $XTRACE 169 | -------------------------------------------------------------------------------- /devstack/plugin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # DevStack extras script to install osprofiler 3 | 4 | # Save trace setting 5 | XTRACE=$(set +o | grep xtrace) 6 | set -o xtrace 7 | 8 | source $DEST/osprofiler/devstack/lib/osprofiler 9 | 10 | if [[ "$1" == "stack" && "$2" == "install" ]]; then 11 | echo_summary "Configuring system services for OSProfiler" 12 | install_osprofiler_collector 13 | 14 | elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then 15 | echo_summary "Configuring OSProfiler" 16 | configure_osprofiler 17 | 18 | elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then 19 | echo_summary "Configuring Tempest" 20 | configure_osprofiler_in_tempest 21 | 22 | elif [[ "$1" == "unstack" ]]; then 23 | if [[ "$OSPROFILER_COLLECTOR" == "jaeger" || \ 24 | "$OSPROFILER_COLLECTOR" == "otlp" ]]; then 25 | echo_summary "Deleting jaeger docker container" 26 | drop_jaeger 27 | fi 28 | fi 29 | 30 | # Restore xtrace 31 | $XTRACE 32 | -------------------------------------------------------------------------------- /devstack/settings: -------------------------------------------------------------------------------- 1 | # Devstack settings 2 | 3 | # A comma-separated list of secrets, that will be used for triggering 4 | # of profiling in OpenStack services: profiling is only performed for 5 | # requests that specify one of these keys in HTTP headers. 6 | OSPROFILER_HMAC_KEYS=${OSPROFILER_HMAC_KEYS:-"SECRET_KEY"} 7 | 8 | # Set whether tracing of SQL requests is enabled or not 9 | OSPROFILER_TRACE_SQLALCHEMY=${OSPROFILER_TRACE_SQLALCHEMY:-"True"} 10 | 11 | enable_service osprofiler 12 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | docutils>=0.14 # OSI-Approved Open Source, Public Domain 2 | openstackdocstheme>=2.2.1 # Apache-2.0 3 | sphinx>=2.0.0,!=2.1.0 # BSD 4 | sphinxcontrib-apidoc>=0.2.1 # BSD 5 | 6 | # Build release notes 7 | reno>=3.1.0 # Apache-2.0 8 | -------------------------------------------------------------------------------- /doc/source/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/osprofiler.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/osprofiler.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/osprofiler" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/osprofiler" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2020 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | # 15 | # OSprofiler documentation build configuration file, created by 16 | # sphinx-quickstart on Fri Jan 10 23:19:18 2014. 17 | # 18 | # This file is execfile()d with the current directory set to its 19 | # containing dir. 20 | # 21 | # Note that not all possible configuration values are present in this 22 | # autogenerated file. 23 | # 24 | # All configuration values have a default; values that are commented out 25 | # serve to show the default. 26 | 27 | import os 28 | import sys 29 | 30 | # If extensions (or modules to document with autodoc) are in another 31 | # directory, add these directories to sys.path here. If the directory 32 | # is relative to the documentation root, use os.path.abspath to make 33 | # it absolute, like shown here. 34 | sys.path.extend([ 35 | os.path.abspath('../..'), 36 | ]) 37 | 38 | 39 | # -- General configuration --------------------- 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 43 | extensions = [ 44 | 'sphinx.ext.autodoc', 45 | 'sphinx.ext.doctest', 46 | 'sphinx.ext.todo', 47 | 'sphinx.ext.coverage', 48 | 'sphinx.ext.ifconfig', 49 | 'sphinx.ext.viewcode', 50 | 'openstackdocstheme', 51 | 'sphinxcontrib.apidoc', 52 | ] 53 | 54 | # openstackdocstheme options 55 | openstackdocs_repo_name = 'openstack/osprofiler' 56 | openstackdocs_auto_name = False 57 | openstackdocs_bug_project = 'osprofiler' 58 | openstackdocs_bug_tag = '' 59 | 60 | todo_include_todos = True 61 | 62 | # Add any paths that contain templates here, relative to this directory. 63 | templates_path = ['_templates'] 64 | 65 | # The suffix of source filenames. 66 | source_suffix = '.rst' 67 | 68 | # The master toctree document. 69 | master_doc = 'index' 70 | 71 | # General information about the project. 72 | project = 'OSprofiler' 73 | copyright = '2016, OpenStack Foundation' 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | exclude_patterns = [] 78 | 79 | # If true, '()' will be appended to :func: etc. cross-reference text. 80 | add_function_parentheses = True 81 | 82 | # If true, the current module name will be prepended to all description 83 | # unit titles (such as .. function::). 84 | add_module_names = True 85 | 86 | # The name of the Pygments (syntax highlighting) style to use. 87 | pygments_style = 'native' 88 | 89 | # -- Options for HTML output ------------ 90 | 91 | # The theme to use for HTML and HTML Help pages. See the documentation for 92 | # a list of builtin themes. 93 | html_theme = 'openstackdocs' 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = [] 99 | 100 | # Output file base name for HTML help builder. 101 | htmlhelp_basename = '%sdoc' % project 102 | 103 | # -- Options for LaTeX output -------------- 104 | 105 | latex_elements = {} 106 | 107 | # Grouping the document tree into LaTeX files. List of tuples 108 | # (source start file, target name, title, author, 109 | # documentclass [howto/manual]). 110 | latex_documents = [ 111 | ('index', 112 | '%s.tex' % project, 113 | '%s Documentation' % project, 114 | 'OpenStack Foundation', 'manual'), 115 | ] 116 | 117 | # -- Options for Texinfo output ----------- 118 | 119 | # Grouping the document tree into Texinfo files. List of tuples 120 | # (source start file, target name, title, author, 121 | # dir menu entry, description, category) 122 | texinfo_documents = [ 123 | ( 124 | 'index', 125 | 'OSprofiler', 126 | 'OSprofiler Documentation', 127 | 'OSprofiler Team', 128 | 'OSprofiler', 129 | 'One line description of project.', 130 | 'Miscellaneous' 131 | ), 132 | ] 133 | 134 | apidoc_output_dir = 'contributor/modules' 135 | apidoc_module_dir = '../../osprofiler' 136 | apidoc_excluded_paths = [ 137 | 'hacking', 138 | 'tests', 139 | ] 140 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | OSProfiler -- Cross-project profiling library 3 | ============================================= 4 | 5 | OSProfiler provides a tiny but powerful library that is used by 6 | most (soon to be all) OpenStack projects and their python clients. It 7 | provides functionality to generate 1 trace per request, that goes 8 | through all involved services. This trace can then be extracted and used 9 | to build a tree of calls which can be quite handy for a variety of 10 | reasons (for example in isolating cross-project performance issues). 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | user/index 16 | 17 | .. toctree:: 18 | :hidden: 19 | 20 | contributor/modules/modules 21 | 22 | 23 | .. rubric:: Indices and tables 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | 29 | -------------------------------------------------------------------------------- /doc/source/user/api.rst: -------------------------------------------------------------------------------- 1 | === 2 | API 3 | === 4 | 5 | There are few things that you should know about API before using it. 6 | 7 | Five ways to add a new trace point. 8 | ----------------------------------- 9 | 10 | .. code-block:: python 11 | 12 | from osprofiler import profiler 13 | 14 | def some_func(): 15 | profiler.start("point_name", {"any_key": "with_any_value"}) 16 | # your code 17 | profiler.stop({"any_info_about_point": "in_this_dict"}) 18 | 19 | 20 | @profiler.trace("point_name", 21 | info={"any_info_about_point": "in_this_dict"}, 22 | hide_args=False) 23 | def some_func2(*args, **kwargs): 24 | # If you need to hide args in profile info, put hide_args=True 25 | pass 26 | 27 | def some_func3(): 28 | with profiler.Trace("point_name", 29 | info={"any_key": "with_any_value"}): 30 | # some code here 31 | 32 | @profiler.trace_cls("point_name", info={}, hide_args=False, 33 | trace_private=False) 34 | class TracedClass(object): 35 | 36 | def traced_method(self): 37 | pass 38 | 39 | def _traced_only_if_trace_private_true(self): 40 | pass 41 | 42 | class RpcManagerClass(object, metaclass=profiler.TracedMeta): 43 | __trace_args__ = {'name': 'rpc', 44 | 'info': None, 45 | 'hide_args': False, 46 | 'trace_private': False} 47 | 48 | def my_method(self, some_args): 49 | pass 50 | 51 | def my_method2(self, some_arg1, some_arg2, kw=None, kw2=None) 52 | pass 53 | 54 | How profiler works? 55 | ------------------- 56 | 57 | * **profiler.Trace()** and **@profiler.trace()** are just syntax sugar, 58 | that just calls **profiler.start()** & **profiler.stop()** methods. 59 | 60 | * Every call of **profiler.start()** & **profiler.stop()** sends to 61 | **collector** 1 message. It means that every trace point creates 2 records 62 | in the collector. *(more about collector & records later)* 63 | 64 | * Nested trace points are supported. The sample below produces 2 trace points: 65 | 66 | .. code-block:: python 67 | 68 | profiler.start("parent_point") 69 | profiler.start("child_point") 70 | profiler.stop() 71 | profiler.stop() 72 | 73 | The implementation is quite simple. Profiler has one stack that contains 74 | ids of all trace points. E.g.: 75 | 76 | .. code-block:: python 77 | 78 | profiler.start("parent_point") # trace_stack.push() 79 | # send to collector -> trace_stack[1] 80 | 81 | profiler.start("child_point") # trace_stack.push() 82 | # send to collector -> trace_stack[2] 83 | profiler.stop() # send to collector -> trace_stack[2] 84 | # trace_stack.pop() 85 | 86 | profiler.stop() # send to collector -> trace_stack[1] 87 | # trace_stack.pop() 88 | 89 | It's simple to build a tree of nested trace points, having 90 | **(parent_id, point_id)** of all trace points. 91 | 92 | Process of sending to collector. 93 | -------------------------------- 94 | 95 | Trace points contain 2 messages (start and stop). Messages like below are 96 | sent to a collector: 97 | 98 | .. parsed-literal:: 99 | 100 | { 101 | "name": -(start|stop) 102 | "base_id": , 103 | "parent_id": , 104 | "trace_id": , 105 | "info": 106 | } 107 | 108 | The fields are defined as the following: 109 | 110 | * base_id - ```` that is equal for all trace points that belong 111 | to one trace, this is done to simplify the process of retrieving 112 | all trace points related to one trace from collector 113 | * parent_id - ```` of parent trace point 114 | * trace_id - ```` of current trace point 115 | * info - the dictionary that contains user information passed when calling 116 | profiler **start()** & **stop()** methods. 117 | 118 | Setting up the collector. 119 | ------------------------- 120 | 121 | Using OSProfiler notifier. 122 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 123 | 124 | .. note:: The following way of configuring OSProfiler is deprecated. The new 125 | version description is located below - `Using OSProfiler initializer.`_. 126 | Don't use OSproliler notifier directly! Its support will be removed soon 127 | from OSProfiler. 128 | 129 | The profiler doesn't include a trace point collector. The user/developer 130 | should instead provide a method that sends messages to a collector. Let's 131 | take a look at a trivial sample, where the collector is just a file: 132 | 133 | .. code-block:: python 134 | 135 | import json 136 | 137 | from osprofiler import notifier 138 | 139 | def send_info_to_file_collector(info, context=None): 140 | with open("traces", "a") as f: 141 | f.write(json.dumps(info)) 142 | 143 | notifier.set(send_info_to_file_collector) 144 | 145 | So now on every **profiler.start()** and **profiler.stop()** call we will 146 | write info about the trace point to the end of the **traces** file. 147 | 148 | Using OSProfiler initializer. 149 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 150 | 151 | OSProfiler now contains various storage drivers to collect tracing data. 152 | Information about what driver to use and what options to pass to OSProfiler 153 | are now stored in OpenStack services configuration files. Example of such 154 | configuration can be found below: 155 | 156 | .. code-block:: bash 157 | 158 | [profiler] 159 | enabled = True 160 | trace_sqlalchemy = True 161 | hmac_keys = SECRET_KEY 162 | connection_string = messaging:// 163 | 164 | If such configuration is provided, OSProfiler setting up can be processed in 165 | following way: 166 | 167 | .. code-block:: python 168 | 169 | if CONF.profiler.enabled: 170 | osprofiler_initializer.init_from_conf( 171 | conf=CONF, 172 | context=context.get_admin_context().to_dict(), 173 | project="cinder", 174 | service=binary, 175 | host=host 176 | ) 177 | 178 | Initialization of profiler. 179 | --------------------------- 180 | 181 | If profiler is not initialized, all calls to **profiler.start()** and 182 | **profiler.stop()** will be ignored. 183 | 184 | Initialization is a quite simple procedure. 185 | 186 | .. code-block:: python 187 | 188 | from osprofiler import profiler 189 | 190 | profiler.init("SECRET_HMAC_KEY", base_id=, parent_id=) 191 | 192 | ``SECRET_HMAC_KEY`` - will be discussed later, because it's related to the 193 | integration of OSprofiler & OpenStack. 194 | 195 | **base_id** and **trace_id** will be used to initialize stack_trace in 196 | profiler, e.g. ``stack_trace = [base_id, trace_id]``. 197 | 198 | OSProfiler CLI. 199 | --------------- 200 | 201 | To make it easier for end users to work with profiler from CLI, OSProfiler 202 | has entry point that allows them to retrieve information about traces and 203 | present it in human readable form. 204 | 205 | Available commands: 206 | 207 | * Help message with all available commands and their arguments: 208 | 209 | .. parsed-literal:: 210 | 211 | $ osprofiler -h/--help 212 | 213 | * OSProfiler version: 214 | 215 | .. parsed-literal:: 216 | 217 | $ osprofiler -v/--version 218 | 219 | * Results of profiling can be obtained in JSON (option: ``--json``) and HTML 220 | (option: ``--html``) formats: 221 | 222 | .. parsed-literal:: 223 | 224 | $ osprofiler trace show --json/--html 225 | 226 | hint: option ``--out`` will redirect result of ``osprofiler trace show`` 227 | in specified file: 228 | 229 | .. parsed-literal:: 230 | 231 | $ osprofiler trace show --json/--html --out /path/to/file 232 | 233 | * In latest versions of OSProfiler with storage drivers (e.g. MongoDB (URI: 234 | ``mongodb://``), Messaging (URI: ``messaging://``)) 235 | ``--connection-string`` parameter should be set up: 236 | 237 | .. parsed-literal:: 238 | 239 | $ osprofiler trace show --connection-string= --json/--html 240 | -------------------------------------------------------------------------------- /doc/source/user/background.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Background 3 | ========== 4 | 5 | OpenStack consists of multiple projects. Each project, in turn, is composed of 6 | multiple services. To process some request, e.g. to boot a virtual machine, 7 | OpenStack uses multiple services from different projects. In the case something 8 | works too slow, it's extremely complicated to understand what exactly goes 9 | wrong and to locate the bottleneck. 10 | 11 | To resolve this issue, we introduce a tiny but powerful library, 12 | **osprofiler**, that is going to be used by all OpenStack projects and their 13 | python clients. It generates 1 trace per request, that goes through 14 | all involved services, and builds a tree of calls. 15 | 16 | Why not cProfile and etc? 17 | ------------------------- 18 | 19 | **The scope of this library is quite different:** 20 | 21 | * We are interested in getting one trace of points from different services, 22 | not tracing all Python calls inside one process. 23 | 24 | * This library should be easy integrable into OpenStack. This means that: 25 | 26 | * It shouldn't require too many changes in code bases of projects it's 27 | integrated with. 28 | 29 | * We should be able to fully turn it off. 30 | 31 | * We should be able to keep it turned on in lazy mode in production 32 | (e.g. admin should be able to "trace" on request). 33 | -------------------------------------------------------------------------------- /doc/source/user/collectors.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Collectors 3 | ========== 4 | 5 | There are a number of drivers to support different collector backends: 6 | 7 | Redis 8 | ----- 9 | 10 | * Overview 11 | 12 | The Redis driver allows profiling data to be collected into a redis 13 | database instance. The traces are stored as key-value pairs where the 14 | key is a string built using trace ids and timestamps and the values 15 | are JSON strings containing the trace information. A second driver is 16 | included to use Redis Sentinel in addition to single node Redis. 17 | 18 | * Capabilities 19 | 20 | * Write trace data to the database. 21 | * Query Traces in database: This allows for pulling trace data 22 | querying on the keys used to save the data in the database. 23 | * Generate a report based on the traces stored in the database. 24 | * Supports use of Redis Sentinel for robustness. 25 | 26 | * Usage 27 | 28 | The driver is used by OSProfiler when using a connection-string URL 29 | of the form redis://[:password]@host[:port][/db]. To use the Sentinel version 30 | use a connection-string of the form 31 | redissentinel://[:password]@host[:port][/db] 32 | 33 | * Configuration 34 | 35 | * No config changes are required by for the base Redis driver. 36 | * There are two configuration options for the Redis Sentinel driver: 37 | 38 | * socket_timeout: specifies the sentinel connection socket timeout 39 | value. Defaults to: 0.1 seconds 40 | * sentinel_service_name: The name of the Sentinel service to use. 41 | Defaults to: "mymaster" 42 | 43 | SQLAlchemy 44 | ---------- 45 | 46 | The SQLAlchemy collector allows you to store profiling data into a database 47 | supported by SQLAlchemy. 48 | 49 | Usage 50 | ===== 51 | To use the driver, the `connection_string` in the `[osprofiler]` config section 52 | needs to be set to a connection string that `SQLAlchemy understands`_ 53 | For example:: 54 | 55 | [osprofiler] 56 | connection_string = mysql+pymysql://username:password@192.168.192.81/profiler?charset=utf8 57 | 58 | where `username` is the database username, `password` is the database password, 59 | `192.168.192.81` is the database IP address and `profiler` is the database name. 60 | 61 | The database (in this example called `profiler`) needs to be created manually and 62 | the database user (in this example called `username`) needs to have priviliges 63 | to create tables and select and insert rows. 64 | 65 | .. note:: 66 | 67 | SQLAlchemy collector requires database JSON data type support. 68 | This type of data is supported by versions listed below or higher: 69 | 70 | - MariaDB 10.2 71 | - MySQL 5.7.8 72 | 73 | .. _SQLAlchemy understands: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls 74 | 75 | 76 | OTLP 77 | ---- 78 | 79 | Use OTLP exporter. Can be used with any comptable backend that support 80 | OTLP. 81 | 82 | Usage 83 | ===== 84 | To use the driver, the `connection_string` in the `[osprofiler]` config section 85 | needs to be set:: 86 | 87 | [osprofiler] 88 | connection_string = otlp://192.168.192.81:4318 89 | 90 | Example: By default, jaeger is listening OTLP on 4318. 91 | 92 | .. note:: 93 | 94 | Curently the exporter is only supporting HTTP. In future some work 95 | may happen to support gRPC. 96 | -------------------------------------------------------------------------------- /doc/source/user/history.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | ChangeLog 3 | ========= 4 | 5 | .. include:: ../../../ChangeLog 6 | -------------------------------------------------------------------------------- /doc/source/user/index.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Using OSProfiler 3 | ================ 4 | 5 | OSProfiler provides a tiny but powerful library that is used by 6 | most (soon to be all) OpenStack projects and their python clients. It 7 | provides functionality to generate 1 trace per request, that goes 8 | through all involved services. This trace can then be extracted and used 9 | to build a tree of calls which can be quite handy for a variety of 10 | reasons (for example in isolating cross-project performance issues). 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | background 16 | api 17 | integration 18 | collectors 19 | similar_projects 20 | 21 | Release Notes 22 | ============= 23 | 24 | .. toctree:: 25 | :maxdepth: 1 26 | 27 | history 28 | -------------------------------------------------------------------------------- /doc/source/user/integration.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Integration 3 | =========== 4 | 5 | There are 4 topics related to integration OSprofiler & `OpenStack`_: 6 | 7 | What we should use as a centralized collector? 8 | ---------------------------------------------- 9 | 10 | We primarily decided to use `Ceilometer`_, because: 11 | 12 | * It's already integrated in OpenStack, so it's quite simple to send 13 | notifications to it from all projects. 14 | 15 | * There is an OpenStack API in Ceilometer that allows us to retrieve all 16 | messages related to one trace. Take a look at 17 | *osprofiler.drivers.ceilometer.Ceilometer:get_report* 18 | 19 | In OSProfiler starting with 1.4.0 version other options (MongoDB driver in 20 | 1.4.0 release, Elasticsearch driver added later, etc.) are also available. 21 | 22 | 23 | How to setup profiler notifier? 24 | ------------------------------- 25 | 26 | We primarily decided to use oslo.messaging Notifier API, because: 27 | 28 | * `oslo.messaging`_ is integrated in all projects 29 | 30 | * It's the simplest way to send notification to Ceilometer, take a 31 | look at: *osprofiler.drivers.messaging.Messaging:notify* method 32 | 33 | * We don't need to add any new `CONF`_ options in projects 34 | 35 | In OSProfiler starting with 1.4.0 version other options (MongoDB driver in 36 | 1.4.0 release, Elasticsearch driver added later, etc.) are also available. 37 | 38 | How to initialize profiler, to get one trace across all services? 39 | ----------------------------------------------------------------- 40 | 41 | To enable cross service profiling we actually need to do send from caller 42 | to callee (base_id & trace_id). So callee will be able to init its profiler 43 | with these values. 44 | 45 | In case of OpenStack there are 2 kinds of interaction between 2 services: 46 | 47 | * REST API 48 | 49 | It's well known that there are python clients for every project, 50 | that generate proper HTTP requests, and parse responses to objects. 51 | 52 | These python clients are used in 2 cases: 53 | 54 | * User access -> OpenStack 55 | 56 | * Service from Project 1 would like to access Service from Project 2 57 | 58 | 59 | So what we need is to: 60 | 61 | * Put in python clients headers with trace info (if profiler is inited) 62 | 63 | * Add `OSprofiler WSGI middleware`_ to your service, this initializes 64 | the profiler, if and only if there are special trace headers, that 65 | are signed by one of the HMAC keys from api-paste.ini (if multiple 66 | keys exist the signing process will continue to use the key that was 67 | accepted during validation). 68 | 69 | * The common items that are used to configure the middleware are the 70 | following (these can be provided when initializing the middleware 71 | object or when setting up the api-paste.ini file):: 72 | 73 | hmac_keys = KEY1, KEY2 (can be a single key as well) 74 | 75 | Actually the algorithm is a bit more complex. The Python client will 76 | also sign the trace info with a `HMAC`_ key (lets call that key ``A``) 77 | passed to profiler.init, and on reception the WSGI middleware will 78 | check that it's signed with *one of* the HMAC keys (the wsgi 79 | server should have key ``A`` as well, but may also have keys ``B`` 80 | and ``C``) that are specified in api-paste.ini. This ensures that only 81 | the user that knows the HMAC key ``A`` in api-paste.ini can init a 82 | profiler properly and send trace info that will be actually 83 | processed. This ensures that trace info that is sent in that 84 | does **not** pass the HMAC validation will be discarded. **NOTE:** The 85 | application of many possible *validation* keys makes it possible to 86 | roll out a key upgrade in a non-impactful manner (by adding a key into 87 | the list and rolling out that change and then removing the older key at 88 | some time in the future). 89 | 90 | * Optionally you can enable client tracing using `requests`_, 91 | Currently only supported by OTLP driver, this will add client call 92 | tracing. see `profiler/trace_requests`'s option. 93 | 94 | * RPC API 95 | 96 | RPC calls are used for interaction between services of one project. 97 | It's well known that projects are using `oslo.messaging`_ to deal with 98 | RPC. It's very good, because projects deal with RPC in similar way. 99 | 100 | So there are 2 required changes: 101 | 102 | * On callee side put in request context trace info (if profiler was 103 | initialized) 104 | 105 | * On caller side initialize profiler, if there is trace info in request 106 | context. 107 | 108 | * Trace all methods of callee API (can be done via profiler.trace_cls). 109 | 110 | 111 | What points should be tracked by default? 112 | ----------------------------------------- 113 | 114 | I think that for all projects we should include by default 5 kinds of points: 115 | 116 | * All HTTP calls - helps to get information about: what HTTP requests were 117 | done, duration of calls (latency of service), information about projects 118 | involved in request. 119 | 120 | * All RPC calls - helps to understand duration of parts of request related 121 | to different services in one project. This information is essential to 122 | understand which service produce the bottleneck. 123 | 124 | * All DB API calls - in some cases slow DB query can produce bottleneck. So 125 | it's quite useful to track how much time request spend in DB layer. 126 | 127 | * All driver calls - in case of nova, cinder and others we have vendor 128 | drivers. Duration 129 | 130 | * ALL SQL requests (turned off by default, because it produce a lot of 131 | traffic) 132 | 133 | .. _CONF: https://docs.openstack.org/oslo.config/latest/ 134 | .. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code 135 | .. _OpenStack: https://www.openstack.org/ 136 | .. _Ceilometer: https://wiki.openstack.org/wiki/Ceilometer 137 | .. _oslo.messaging: https://pypi.org/project/oslo.messaging 138 | .. _OSprofiler WSGI middleware: https://github.com/openstack/osprofiler/blob/master/osprofiler/web.py 139 | .. _requests: https://docs.python-requests.org/en/latest/index.html 140 | -------------------------------------------------------------------------------- /doc/source/user/similar_projects.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Similar projects 3 | ================ 4 | 5 | Other projects (some alive, some abandoned, some research prototypes) 6 | that are similar (in idea and ideal to OSprofiler). 7 | 8 | * `Zipkin`_ 9 | * `Dapper`_ 10 | * `Tomograph`_ 11 | * `HTrace`_ 12 | * `Jaeger`_ 13 | * `OpenTracing`_ 14 | 15 | .. _Zipkin: https://zipkin.io/ 16 | .. _Dapper: http://research.google.com/pubs/pub36356.html 17 | .. _Tomograph: https://github.com/stackforge/tomograph 18 | .. _HTrace: https://htrace.incubator.apache.org/ 19 | .. _Jaeger: https://uber.github.io/jaeger/ 20 | .. _OpenTracing: https://opentracing.io/ 21 | -------------------------------------------------------------------------------- /doc/specs/README.rst: -------------------------------------------------------------------------------- 1 | OSProfiler Specs 2 | ================ 3 | 4 | Specs are detailed description of proposed changes in project. Usually they 5 | answer on what, why, how to change in project and who is going to work on 6 | change. 7 | 8 | This directory contains 2 subdirectories: 9 | 10 | - in-progress - These specs are approved, but they are not implemented yet 11 | - implemented - Implemented specs archive 12 | -------------------------------------------------------------------------------- /doc/specs/implemented/README.rst: -------------------------------------------------------------------------------- 1 | OSprofiler Implemented Specs 2 | ============================ 3 | 4 | Specs are detailed description of proposed changes in project. Usually they 5 | answer on what, why, how to change in project and who is going to work on 6 | change. 7 | 8 | This directory contains files with implemented specs, 1 file is 1 spec. 9 | -------------------------------------------------------------------------------- /doc/specs/implemented/make_paste_ini_config_optional.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This work is licensed under a Creative Commons Attribution 3.0 Unported 3 | License. 4 | 5 | http://creativecommons.org/licenses/by/3.0/legalcode 6 | 7 | .. 8 | This template should be in ReSTructured text. The filename in the git 9 | repository should match the launchpad URL, for example a URL of 10 | https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named 11 | awesome-thing.rst . Please do not delete any of the sections in this 12 | template. If you have nothing to say for a whole section, just write: None 13 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 14 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 15 | 16 | ====================================== 17 | Make api-paste.ini Arguments Optional 18 | ====================================== 19 | 20 | Problem description 21 | =================== 22 | 23 | Integration of OSprofiler with OpenStack projects is harder than it should be, 24 | it requires keeping part of arguments inside api-paste.ini files and part in 25 | projects.conf file. 26 | 27 | We should make all configuration options from api-paste.ini file optional 28 | and add alternative way to configure osprofiler.web.WsgiMiddleware 29 | 30 | 31 | Proposed change 32 | =============== 33 | 34 | Integration of OSprofiler requires 2 changes in api-paste.ini file: 35 | 36 | - One is adding osprofiler.web.WsgiMiddleware to pipelines: 37 | https://github.com/openstack/cinder/blob/master/etc/cinder/api-paste.ini#L13 38 | 39 | - Another is to add it's arguments: 40 | https://github.com/openstack/cinder/blob/master/etc/cinder/api-paste.ini#L31-L32 41 | 42 | so WsgiMiddleware will be correctly initialized here: 43 | https://github.com/openstack/osprofiler/blob/51761f375189bdc03b7e72a266ad0950777f32b1/osprofiler/web.py#L64 44 | 45 | We should make ``hmac_keys`` and ``enabled`` variable optional, create 46 | separated method from initialization of wsgi middleware and cut new release. 47 | After that remove 48 | 49 | 50 | Alternatives 51 | ------------ 52 | 53 | None. 54 | 55 | 56 | Implementation 57 | ============== 58 | 59 | Assignee(s) 60 | ----------- 61 | 62 | Primary assignee: 63 | dbelova 64 | 65 | Work Items 66 | ---------- 67 | 68 | - Modify osprofiler.web.WsgiMiddleware to make ``hmac_keys`` optional (done) 69 | 70 | - Add alternative way to setup osprofiler.web.WsgiMiddleware, e.g. extra 71 | argument hmac_keys to enable() method (done) 72 | 73 | - Cut new release 0.3.1 (tbd) 74 | 75 | - Fix the code in all projects: remove api-paste.ini arguments and use 76 | osprofiler.web.enable with extra argument (tbd) 77 | 78 | 79 | Dependencies 80 | ============ 81 | 82 | - Cinder, Glance, Trove - projects should be fixed 83 | -------------------------------------------------------------------------------- /doc/specs/implemented/multi_backend_support.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This work is licensed under a Creative Commons Attribution 3.0 Unported 3 | License. 4 | 5 | http://creativecommons.org/licenses/by/3.0/legalcode 6 | 7 | .. 8 | This template should be in ReSTructured text. The filename in the git 9 | repository should match the launchpad URL, for example a URL of 10 | https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named 11 | awesome-thing.rst . Please do not delete any of the sections in this 12 | template. If you have nothing to say for a whole section, just write: None 13 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 14 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 15 | 16 | ===================== 17 | Multi backend support 18 | ===================== 19 | 20 | Make OSProfiler more flexible and production ready. 21 | 22 | Problem description 23 | =================== 24 | 25 | Currently OSprofiler works only with one backend Ceilometer which actually 26 | doesn't work well and adds huge overhead. More over often Ceilometer is not 27 | installed/used at all. To resolve this we should add support for different 28 | backends like: MongoDB, InfluxDB, ElasticSearch, ... 29 | 30 | 31 | Proposed change 32 | =============== 33 | 34 | And new osprofiler.drivers mechanism, each driver will do 2 things: 35 | send notifications and parse all notification in unified tree structure 36 | that can be processed by the REST lib. 37 | 38 | Deprecate osprofiler.notifiers and osprofiler.parsers 39 | 40 | Change all projects that are using OSprofiler to new model 41 | 42 | Alternatives 43 | ------------ 44 | 45 | I don't know any good alternative. 46 | 47 | Implementation 48 | ============== 49 | 50 | Assignee(s) 51 | ----------- 52 | 53 | Primary assignees: 54 | dbelova 55 | ayelistratov 56 | 57 | 58 | Work Items 59 | ---------- 60 | 61 | To add support of multi backends we should change few places in osprofiler 62 | that are hardcoded on Ceilometer: 63 | 64 | - CLI command ``show``: 65 | 66 | I believe we should add extra argument "connection_string" which will allow 67 | people to specify where is backend. So it will look like: 68 | ://[[user[:password]]@[address][:port][/database]] 69 | 70 | - Merge osprofiler.notifiers and osprofiler.parsers to osprofiler.drivers 71 | 72 | Notifiers and Parsers are tightly related. Like for MongoDB notifier you 73 | should use MongoDB parsers, so there is better solution to keep both 74 | in the same place. 75 | 76 | This change should be done with keeping backward compatibility, 77 | in other words 78 | we should create separated directory osprofiler.drivers and put first 79 | Ceilometer and then start working on other backends. 80 | 81 | These drivers will be chosen based on connection string 82 | 83 | - Deprecate osprofiler.notifiers and osprofiler.parsers 84 | 85 | - Switch all projects to new model with connection string 86 | 87 | 88 | Dependencies 89 | ============ 90 | 91 | - Cinder, Glance, Trove, Heat should be changed 92 | -------------------------------------------------------------------------------- /doc/specs/in-progress/README.rst: -------------------------------------------------------------------------------- 1 | OSprofiler In-Progress Specs 2 | ============================ 3 | 4 | Specs are detailed description of proposed changes in project. Usually they 5 | answer on what, why, how to change in project and who is going to work on 6 | change. 7 | 8 | This directory contains files with accepted by not implemented specs, 9 | 1 file is 1 spec. 10 | -------------------------------------------------------------------------------- /doc/specs/in-progress/better_devstack_integration.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This work is licensed under a Creative Commons Attribution 3.0 Unported 3 | License. 4 | 5 | http://creativecommons.org/licenses/by/3.0/legalcode 6 | 7 | .. 8 | This template should be in ReSTructured text. The filename in the git 9 | repository should match the launchpad URL, for example a URL of 10 | https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named 11 | awesome-thing.rst . Please do not delete any of the sections in this 12 | template. If you have nothing to say for a whole section, just write: None 13 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 14 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 15 | 16 | ============================ 17 | Better DevStack Integration 18 | ============================ 19 | 20 | Make it simple to enable OSprofiler like it is simple to enable DEBUG log level 21 | 22 | Problem description 23 | =================== 24 | 25 | It's hard to turn on OSProfiler in DevStack, you have to change 26 | notification_topic and enable Ceilometer and in future do other magic. 27 | As well if something is done wrong it's hard to debug 28 | 29 | 30 | Proposed change 31 | =============== 32 | 33 | Make a single argument: PROFILING=True/False 34 | 35 | Alternatives 36 | ------------ 37 | 38 | Do nothing and keep things hard. 39 | 40 | Implementation 41 | ============== 42 | 43 | Assignee(s) 44 | ----------- 45 | 46 | Primary assignee: 47 | boris-42 48 | 49 | 50 | Work Items 51 | ---------- 52 | 53 | - Make DevStack plugin for OSprofiler 54 | 55 | - Configure Ceilometer 56 | 57 | - Configure services that support OSprofiler 58 | 59 | 60 | Dependencies 61 | ============ 62 | 63 | - DevStack 64 | -------------------------------------------------------------------------------- /doc/specs/in-progress/integration_testing.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This work is licensed under a Creative Commons Attribution 3.0 Unported 3 | License. 4 | 5 | http://creativecommons.org/licenses/by/3.0/legalcode 6 | 7 | .. 8 | This template should be in ReSTructured text. The filename in the git 9 | repository should match the launchpad URL, for example a URL of 10 | https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named 11 | awesome-thing.rst . Please do not delete any of the sections in this 12 | template. If you have nothing to say for a whole section, just write: None 13 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 14 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 15 | 16 | =================== 17 | Integration Testing 18 | =================== 19 | 20 | We should create DSVM job that check that proposed changes in OSprofiler 21 | don't break projects that are using OSProfiler. 22 | 23 | 24 | Problem description 25 | =================== 26 | 27 | Currently we don't have CI for testing that OSprofiler changes are backward 28 | compatible and don't break projects that are using OSprofiler. In other words 29 | without this job each time when we are releasing OSProfiler we can break 30 | some of OpenStack projects which is quite bad. 31 | 32 | Proposed change 33 | =============== 34 | 35 | Create DSVM job that will install OSprofiler with proposed patch instead of 36 | the latest release and run some basic tests. 37 | 38 | Alternatives 39 | ------------ 40 | 41 | Do nothing and break the OpenStack.. 42 | 43 | Implementation 44 | ============== 45 | 46 | Assignee(s) 47 | ----------- 48 | 49 | Primary assignee: 50 | 51 | 52 | 53 | Work Items 54 | ---------- 55 | 56 | - Create DSVM job 57 | - Run Rally tests to make sure that everything works 58 | 59 | 60 | Dependencies 61 | ============ 62 | 63 | None 64 | -------------------------------------------------------------------------------- /doc/specs/template.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This work is licensed under a Creative Commons Attribution 3.0 Unported 3 | License. 4 | 5 | http://creativecommons.org/licenses/by/3.0/legalcode 6 | 7 | .. 8 | This template should be in ReSTructured text. The filename in the git 9 | repository should match the launchpad URL, for example a URL of 10 | https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named 11 | awesome-thing.rst . Please do not delete any of the sections in this 12 | template. If you have nothing to say for a whole section, just write: None 13 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 14 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 15 | 16 | ======================= 17 | The title of your Spec 18 | ======================= 19 | 20 | Introduction paragraph -- why are we doing anything? 21 | 22 | Problem description 23 | =================== 24 | 25 | A detailed description of the problem. 26 | 27 | Proposed change 28 | =============== 29 | 30 | Here is where you cover the change you propose to make in detail. How do you 31 | propose to solve this problem? 32 | 33 | If this is one part of a larger effort make it clear where this piece ends. In 34 | other words, what's the scope of this effort? 35 | 36 | Include where in the heat tree hierarchy this will reside. 37 | 38 | Alternatives 39 | ------------ 40 | 41 | This is an optional section, where it does apply we'd just like a demonstration 42 | that some thought has been put into why the proposed approach is the best one. 43 | 44 | Implementation 45 | ============== 46 | 47 | Assignee(s) 48 | ----------- 49 | 50 | Who is leading the writing of the code? Or is this a blueprint where you're 51 | throwing it out there to see who picks it up? 52 | 53 | If more than one person is working on the implementation, please designate the 54 | primary author and contact. 55 | 56 | Primary assignee: 57 | 58 | 59 | Can optionally can list additional ids if they intend on doing 60 | substantial implementation work on this blueprint. 61 | 62 | Work Items 63 | ---------- 64 | 65 | Work items or tasks -- break the feature up into the things that need to be 66 | done to implement it. Those parts might end up being done by different people, 67 | but we're mostly trying to understand the timeline for implementation. 68 | 69 | 70 | Dependencies 71 | ============ 72 | 73 | - Include specific references to specs and/or blueprints in heat, or in other 74 | projects, that this one either depends on or is related to. 75 | 76 | - Does this feature require any new library dependencies or code otherwise not 77 | included in OpenStack? Or does it depend on a specific version of library? 78 | 79 | -------------------------------------------------------------------------------- /osprofiler/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 OpenStack Foundation. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import importlib.metadata 17 | 18 | __version__ = importlib.metadata.version("osprofiler") 19 | -------------------------------------------------------------------------------- /osprofiler/_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import base64 17 | import hashlib 18 | import hmac 19 | import json 20 | import os 21 | import uuid 22 | 23 | from oslo_utils import uuidutils 24 | 25 | 26 | def split(text, strip=True): 27 | """Splits a comma separated text blob into its components. 28 | 29 | Does nothing if already a list or tuple. 30 | """ 31 | if isinstance(text, (tuple, list)): 32 | return text 33 | if not isinstance(text, str): 34 | raise TypeError( 35 | "Unknown how to split '{}': {}".format(text, type(text))) 36 | if strip: 37 | return [t.strip() for t in text.split(",") if t.strip()] 38 | else: 39 | return text.split(",") 40 | 41 | 42 | def binary_encode(text, encoding="utf-8"): 43 | """Converts a string of into a binary type using given encoding. 44 | 45 | Does nothing if text not unicode string. 46 | """ 47 | if isinstance(text, bytes): 48 | return text 49 | elif isinstance(text, str): 50 | return text.encode(encoding) 51 | else: 52 | raise TypeError("Expected binary or string type") 53 | 54 | 55 | def binary_decode(data, encoding="utf-8"): 56 | """Converts a binary type into a text type using given encoding. 57 | 58 | Does nothing if data is already unicode string. 59 | """ 60 | if isinstance(data, bytes): 61 | return data.decode(encoding) 62 | elif isinstance(data, str): 63 | return data 64 | else: 65 | raise TypeError("Expected binary or string type") 66 | 67 | 68 | def generate_hmac(data, hmac_key): 69 | """Generate a hmac using a known key given the provided content.""" 70 | h = hmac.new(binary_encode(hmac_key), digestmod=hashlib.sha1) 71 | h.update(binary_encode(data)) 72 | return h.hexdigest() 73 | 74 | 75 | def signed_pack(data, hmac_key): 76 | """Pack and sign data with hmac_key.""" 77 | raw_data = base64.urlsafe_b64encode(binary_encode(json.dumps(data))) 78 | 79 | # NOTE(boris-42): Don't generate_hmac if there is no hmac_key, mostly 80 | # security reason, we shouldn't allow to use WsgiMiddleware 81 | # without hmac_key, cause everybody will be able to trigger 82 | # profiler and organize DDOS. 83 | return raw_data, generate_hmac(raw_data, hmac_key) if hmac_key else None 84 | 85 | 86 | def signed_unpack(data, hmac_data, hmac_keys): 87 | """Unpack data and check that it was signed with hmac_key. 88 | 89 | :param data: json string that was singed_packed. 90 | :param hmac_data: hmac data that was generated from json by hmac_key on 91 | user side 92 | :param hmac_keys: server side hmac_keys, one of these should be the same 93 | as user used to sign with 94 | 95 | :returns: None in case of something wrong, Object in case of everything OK. 96 | """ 97 | # NOTE(boris-42): For security reason, if there is no hmac_data or 98 | # hmac_keys we don't trust data => return None. 99 | if not (hmac_keys and hmac_data): 100 | return None 101 | hmac_data = hmac_data.strip() 102 | if not hmac_data: 103 | return None 104 | for hmac_key in hmac_keys: 105 | try: 106 | user_hmac_data = generate_hmac(data, hmac_key) 107 | except Exception: # nosec 108 | pass 109 | else: 110 | if hmac.compare_digest(hmac_data, user_hmac_data): 111 | try: 112 | contents = json.loads( 113 | binary_decode(base64.urlsafe_b64decode(data))) 114 | contents["hmac_key"] = hmac_key 115 | return contents 116 | except Exception: 117 | return None 118 | return None 119 | 120 | 121 | def itersubclasses(cls, _seen=None): 122 | """Generator over all subclasses of a given class in depth first order.""" 123 | 124 | _seen = _seen or set() 125 | try: 126 | subs = cls.__subclasses__() 127 | except TypeError: # fails only when cls is type 128 | subs = cls.__subclasses__(cls) 129 | for sub in subs: 130 | if sub not in _seen: 131 | _seen.add(sub) 132 | yield sub 133 | for sub in itersubclasses(sub, _seen): 134 | yield sub 135 | 136 | 137 | def import_modules_from_package(package): 138 | """Import modules from package and append into sys.modules 139 | 140 | :param: package - Full package name. For example: rally.deploy.engines 141 | """ 142 | path = [os.path.dirname(__file__), ".."] + package.split(".") 143 | path = os.path.join(*path) 144 | for root, dirs, files in os.walk(path): 145 | for filename in files: 146 | if filename.startswith("__") or not filename.endswith(".py"): 147 | continue 148 | new_package = ".".join(root.split(os.sep)).split("....")[1] 149 | module_name = "{}.{}".format(new_package, filename[:-3]) 150 | __import__(module_name) 151 | 152 | 153 | def shorten_id(span_id): 154 | """Convert from uuid4 to 64 bit id for OpenTracing""" 155 | int64_max = (1 << 64) - 1 156 | if isinstance(span_id, int): 157 | return span_id & int64_max 158 | try: 159 | short_id = uuid.UUID(span_id).int & int64_max 160 | except ValueError: 161 | # Return a new short id for this 162 | short_id = shorten_id(uuidutils.generate_uuid()) 163 | return short_id 164 | 165 | 166 | def uuid_to_int128(span_uuid): 167 | """Convert from uuid4 to 128 bit id for OpenTracing""" 168 | if isinstance(span_uuid, int): 169 | return span_uuid 170 | try: 171 | span_int = uuid.UUID(span_uuid).int 172 | except ValueError: 173 | # Return a new short id for this 174 | span_int = uuid_to_int128(uuidutils.generate_uuid()) 175 | return span_int 176 | -------------------------------------------------------------------------------- /osprofiler/cmd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/cmd/__init__.py -------------------------------------------------------------------------------- /osprofiler/cmd/cliutils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import os 17 | 18 | 19 | def env(*args, **kwargs): 20 | """Returns the first environment variable set. 21 | 22 | If all are empty, defaults to '' or keyword arg `default`. 23 | """ 24 | for arg in args: 25 | value = os.environ.get(arg) 26 | if value: 27 | return value 28 | return kwargs.get("default", "") 29 | 30 | 31 | def arg(*args, **kwargs): 32 | """Decorator for CLI args. 33 | 34 | Example: 35 | 36 | >>> @arg("name", help="Name of the new entity") 37 | ... def entity_create(args): 38 | ... pass 39 | """ 40 | def _decorator(func): 41 | add_arg(func, *args, **kwargs) 42 | return func 43 | return _decorator 44 | 45 | 46 | def add_arg(func, *args, **kwargs): 47 | """Bind CLI arguments to a shell.py `do_foo` function.""" 48 | 49 | if not hasattr(func, "arguments"): 50 | func.arguments = [] 51 | 52 | # NOTE(sirp): avoid dups that can occur when the module is shared across 53 | # tests. 54 | if (args, kwargs) not in func.arguments: 55 | # Because of the semantics of decorator composition if we just append 56 | # to the options list positional options will appear to be backwards. 57 | func.arguments.insert(0, (args, kwargs)) 58 | -------------------------------------------------------------------------------- /osprofiler/cmd/commands.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import json 17 | import os 18 | 19 | from oslo_utils import encodeutils 20 | from oslo_utils import uuidutils 21 | import prettytable 22 | 23 | from osprofiler.cmd import cliutils 24 | from osprofiler.drivers import base 25 | from osprofiler import exc 26 | 27 | 28 | class BaseCommand: 29 | group_name = None 30 | 31 | 32 | class TraceCommands(BaseCommand): 33 | group_name = "trace" 34 | 35 | @cliutils.arg("trace", help="File with trace or trace id") 36 | @cliutils.arg("--connection-string", dest="conn_str", 37 | default=(cliutils.env("OSPROFILER_CONNECTION_STRING")), 38 | help="Storage driver's connection string. Defaults to " 39 | "env[OSPROFILER_CONNECTION_STRING] if set") 40 | @cliutils.arg("--transport-url", dest="transport_url", 41 | help="Oslo.messaging transport URL (for messaging:// driver " 42 | "only), e.g. rabbit://user:password@host:5672/") 43 | @cliutils.arg("--idle-timeout", dest="idle_timeout", type=int, default=1, 44 | help="How long to wait for the trace to finish, in seconds " 45 | "(for messaging:// driver only)") 46 | @cliutils.arg("--json", dest="use_json", action="store_true", 47 | help="show trace in JSON") 48 | @cliutils.arg("--html", dest="use_html", action="store_true", 49 | help="show trace in HTML") 50 | @cliutils.arg("--local-libs", dest="local_libs", action="store_true", 51 | help="use local static files of html in /libs/") 52 | @cliutils.arg("--dot", dest="use_dot", action="store_true", 53 | help="show trace in DOT language") 54 | @cliutils.arg("--render-dot", dest="render_dot_filename", 55 | help="filename for rendering the dot graph in pdf format") 56 | @cliutils.arg("--out", dest="file_name", help="save output in file") 57 | def show(self, args): 58 | """Display trace results in HTML, JSON or DOT format.""" 59 | 60 | if not args.conn_str: 61 | raise exc.CommandError( 62 | "You must provide connection string via" 63 | " either --connection-string or " 64 | "via env[OSPROFILER_CONNECTION_STRING]") 65 | 66 | trace = None 67 | 68 | if not uuidutils.is_uuid_like(args.trace): 69 | trace = json.load(open(args.trace)) 70 | else: 71 | try: 72 | engine = base.get_driver(args.conn_str, **args.__dict__) 73 | except Exception as e: 74 | raise exc.CommandError(e.message) 75 | 76 | trace = engine.get_report(args.trace) 77 | 78 | if not trace or not trace.get("children"): 79 | msg = ("Trace with UUID %s not found. Please check the HMAC key " 80 | "used in the command." % args.trace) 81 | raise exc.CommandError(msg) 82 | 83 | # Since datetime.datetime is not JSON serializable by default, 84 | # this method will handle that. 85 | def datetime_json_serialize(obj): 86 | if hasattr(obj, "isoformat"): 87 | return obj.isoformat() 88 | else: 89 | return obj 90 | 91 | if args.use_json: 92 | output = json.dumps(trace, default=datetime_json_serialize, 93 | separators=(",", ": "), 94 | indent=2) 95 | elif args.use_html: 96 | with open(os.path.join(os.path.dirname(__file__), 97 | "template.html")) as html_template: 98 | output = html_template.read().replace( 99 | "$DATA", json.dumps(trace, indent=4, 100 | separators=(",", ": "), 101 | default=datetime_json_serialize)) 102 | if args.local_libs: 103 | output = output.replace("$LOCAL", "true") 104 | else: 105 | output = output.replace("$LOCAL", "false") 106 | elif args.use_dot: 107 | dot_graph = self._create_dot_graph(trace) 108 | output = dot_graph.source 109 | if args.render_dot_filename: 110 | dot_graph.render(args.render_dot_filename, cleanup=True) 111 | else: 112 | raise exc.CommandError("You should choose one of the following " 113 | "output formats: json, html or dot.") 114 | 115 | if args.file_name: 116 | with open(args.file_name, "w+") as output_file: 117 | output_file.write(output) 118 | else: 119 | print(output) 120 | 121 | def _create_dot_graph(self, trace): 122 | try: 123 | import graphviz 124 | except ImportError: 125 | raise exc.CommandError( 126 | "graphviz library is required to use this option.") 127 | 128 | dot = graphviz.Digraph(format="pdf") 129 | next_id = [0] 130 | 131 | def _create_node(info): 132 | time_taken = info["finished"] - info["started"] 133 | service = info["service"] + ":" if "service" in info else "" 134 | name = info["name"] 135 | label = "%s%s - %d ms" % (service, name, time_taken) 136 | 137 | if name == "wsgi": 138 | req = info["meta.raw_payload.wsgi-start"]["info"]["request"] 139 | label = "{}\\n{} {}..".format(label, req["method"], 140 | req["path"][:30]) 141 | elif name == "rpc" or name == "driver": 142 | raw = info["meta.raw_payload.%s-start" % name] 143 | fn_name = raw["info"]["function"]["name"] 144 | label = "{}\\n{}".format(label, fn_name.split(".")[-1]) 145 | 146 | node_id = str(next_id[0]) 147 | next_id[0] += 1 148 | dot.node(node_id, label) 149 | return node_id 150 | 151 | def _create_sub_graph(root): 152 | rid = _create_node(root["info"]) 153 | for child in root["children"]: 154 | cid = _create_sub_graph(child) 155 | dot.edge(rid, cid) 156 | return rid 157 | 158 | _create_sub_graph(trace) 159 | return dot 160 | 161 | @cliutils.arg("--connection-string", dest="conn_str", 162 | default=cliutils.env("OSPROFILER_CONNECTION_STRING"), 163 | help="Storage driver's connection string. Defaults to " 164 | "env[OSPROFILER_CONNECTION_STRING] if set") 165 | @cliutils.arg("--error-trace", dest="error_trace", 166 | type=bool, default=False, 167 | help="List all traces that contain error.") 168 | def list(self, args): 169 | """List all traces""" 170 | if not args.conn_str: 171 | raise exc.CommandError( 172 | "You must provide connection string via" 173 | " either --connection-string or " 174 | "via env[OSPROFILER_CONNECTION_STRING]") 175 | try: 176 | engine = base.get_driver(args.conn_str, **args.__dict__) 177 | except Exception as e: 178 | raise exc.CommandError(e.message) 179 | 180 | fields = ("base_id", "timestamp") 181 | pretty_table = prettytable.PrettyTable(fields) 182 | pretty_table.align = "l" 183 | if not args.error_trace: 184 | traces = engine.list_traces(fields) 185 | else: 186 | traces = engine.list_error_traces() 187 | for trace in traces: 188 | row = [trace[field] for field in fields] 189 | pretty_table.add_row(row) 190 | print(encodeutils.safe_encode(pretty_table.get_string()).decode()) 191 | -------------------------------------------------------------------------------- /osprofiler/cmd/shell.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | 17 | """ 18 | Command-line interface to the OpenStack Profiler. 19 | """ 20 | 21 | import argparse 22 | import inspect 23 | import sys 24 | 25 | from oslo_config import cfg 26 | 27 | import osprofiler 28 | from osprofiler.cmd import commands 29 | from osprofiler import exc 30 | from osprofiler import opts 31 | 32 | 33 | class OSProfilerShell: 34 | 35 | def __init__(self, argv): 36 | args = self._get_base_parser().parse_args(argv) 37 | opts.set_defaults(cfg.CONF) 38 | 39 | args.func(args) 40 | 41 | def _get_base_parser(self): 42 | parser = argparse.ArgumentParser( 43 | prog="osprofiler", 44 | description=__doc__.strip(), 45 | add_help=True 46 | ) 47 | 48 | parser.add_argument("-v", "--version", 49 | action="version", 50 | version=osprofiler.__version__) 51 | 52 | self._append_subcommands(parser) 53 | 54 | return parser 55 | 56 | def _append_subcommands(self, parent_parser): 57 | subcommands = parent_parser.add_subparsers(help="") 58 | for group_cls in commands.BaseCommand.__subclasses__(): 59 | group_parser = subcommands.add_parser(group_cls.group_name) 60 | subcommand_parser = group_parser.add_subparsers() 61 | 62 | for name, callback in inspect.getmembers( 63 | group_cls(), predicate=inspect.ismethod): 64 | command = name.replace("_", "-") 65 | desc = callback.__doc__ or "" 66 | help_message = desc.strip().split("\n")[0] 67 | arguments = getattr(callback, "arguments", []) 68 | 69 | command_parser = subcommand_parser.add_parser( 70 | command, help=help_message, description=desc) 71 | for (args, kwargs) in arguments: 72 | command_parser.add_argument(*args, **kwargs) 73 | command_parser.set_defaults(func=callback) 74 | 75 | def _no_project_and_domain_set(self, args): 76 | if not (args.os_project_id or (args.os_project_name 77 | and (args.os_user_domain_name or args.os_user_domain_id)) 78 | or (args.os_tenant_id or args.os_tenant_name)): 79 | return True 80 | else: 81 | return False 82 | 83 | 84 | def main(args=None): 85 | if args is None: 86 | args = sys.argv[1:] 87 | 88 | try: 89 | OSProfilerShell(args) 90 | except exc.CommandError as e: 91 | print(e.message) 92 | return 1 93 | 94 | 95 | if __name__ == "__main__": 96 | main() 97 | -------------------------------------------------------------------------------- /osprofiler/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | from osprofiler.drivers import base # noqa 2 | from osprofiler.drivers import elasticsearch_driver # noqa 3 | from osprofiler.drivers import jaeger # noqa 4 | from osprofiler.drivers import otlp # noqa 5 | from osprofiler.drivers import loginsight # noqa 6 | from osprofiler.drivers import messaging # noqa 7 | from osprofiler.drivers import mongodb # noqa 8 | from osprofiler.drivers import redis_driver # noqa 9 | from osprofiler.drivers import sqlalchemy_driver # noqa 10 | -------------------------------------------------------------------------------- /osprofiler/drivers/elasticsearch_driver.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from urllib import parse as parser 17 | 18 | from oslo_config import cfg 19 | 20 | from osprofiler.drivers import base 21 | from osprofiler import exc 22 | 23 | 24 | class ElasticsearchDriver(base.Driver): 25 | def __init__(self, connection_str, index_name="osprofiler-notifications", 26 | project=None, service=None, host=None, conf=cfg.CONF, 27 | **kwargs): 28 | """Elasticsearch driver for OSProfiler.""" 29 | 30 | super().__init__(connection_str, 31 | project=project, 32 | service=service, 33 | host=host, 34 | conf=conf, 35 | **kwargs) 36 | try: 37 | from elasticsearch import Elasticsearch 38 | except ImportError: 39 | raise exc.CommandError( 40 | "To use OSProfiler with ElasticSearch driver, " 41 | "please install `elasticsearch` library. " 42 | "To install with pip:\n `pip install elasticsearch`.") 43 | 44 | client_url = parser.urlunparse(parser.urlparse(self.connection_str) 45 | ._replace(scheme="http")) 46 | self.conf = conf 47 | self.client = Elasticsearch(client_url) 48 | self.index_name = index_name 49 | self.index_name_error = "osprofiler-notifications-error" 50 | 51 | @classmethod 52 | def get_name(cls): 53 | return "elasticsearch" 54 | 55 | def notify(self, info): 56 | """Send notifications to Elasticsearch. 57 | 58 | :param info: Contains information about trace element. 59 | In payload dict there are always 3 ids: 60 | "base_id" - uuid that is common for all notifications 61 | related to one trace. Used to simplify retrieving of all 62 | trace elements from Elasticsearch. 63 | "parent_id" - uuid of parent element in trace 64 | "trace_id" - uuid of current element in trace 65 | With parent_id and trace_id it's quite simple to build 66 | tree of trace elements, which simplify analyze of trace. 67 | """ 68 | 69 | info = info.copy() 70 | info["project"] = self.project 71 | info["service"] = self.service 72 | self.client.index(index=self.index_name, 73 | doc_type=self.conf.profiler.es_doc_type, body=info) 74 | 75 | if (self.filter_error_trace 76 | and info.get("info", {}).get("etype") is not None): 77 | self.notify_error_trace(info) 78 | 79 | def notify_error_trace(self, info): 80 | """Store base_id and timestamp of error trace to a separate index.""" 81 | self.client.index( 82 | index=self.index_name_error, 83 | doc_type=self.conf.profiler.es_doc_type, 84 | body={"base_id": info["base_id"], "timestamp": info["timestamp"]} 85 | ) 86 | 87 | def _hits(self, response): 88 | """Returns all hits of search query using scrolling 89 | 90 | :param response: ElasticSearch query response 91 | """ 92 | scroll_id = response["_scroll_id"] 93 | scroll_size = len(response["hits"]["hits"]) 94 | result = [] 95 | 96 | while scroll_size > 0: 97 | for hit in response["hits"]["hits"]: 98 | result.append(hit["_source"]) 99 | response = self.client.scroll(scroll_id=scroll_id, 100 | scroll=self.conf.profiler. 101 | es_scroll_time) 102 | scroll_id = response["_scroll_id"] 103 | scroll_size = len(response["hits"]["hits"]) 104 | 105 | return result 106 | 107 | def list_traces(self, fields=None): 108 | """Query all traces from the storage. 109 | 110 | :param fields: Set of trace fields to return. Defaults to 'base_id' 111 | and 'timestamp' 112 | :returns: List of traces, where each trace is a dictionary containing 113 | at least `base_id` and `timestamp`. 114 | """ 115 | query = {"match_all": {}} 116 | fields = set(fields or self.default_trace_fields) 117 | 118 | response = self.client.search(index=self.index_name, 119 | doc_type=self.conf.profiler.es_doc_type, 120 | size=self.conf.profiler.es_scroll_size, 121 | scroll=self.conf.profiler.es_scroll_time, 122 | body={"_source": fields, "query": query, 123 | "sort": [{"timestamp": "asc"}]}) 124 | 125 | return self._hits(response) 126 | 127 | def list_error_traces(self): 128 | """Returns all traces that have error/exception.""" 129 | response = self.client.search( 130 | index=self.index_name_error, 131 | doc_type=self.conf.profiler.es_doc_type, 132 | size=self.conf.profiler.es_scroll_size, 133 | scroll=self.conf.profiler.es_scroll_time, 134 | body={ 135 | "_source": self.default_trace_fields, 136 | "query": {"match_all": {}}, 137 | "sort": [{"timestamp": "asc"}] 138 | } 139 | ) 140 | 141 | return self._hits(response) 142 | 143 | def get_report(self, base_id): 144 | """Retrieves and parses notification from Elasticsearch. 145 | 146 | :param base_id: Base id of trace elements. 147 | """ 148 | response = self.client.search(index=self.index_name, 149 | doc_type=self.conf.profiler.es_doc_type, 150 | size=self.conf.profiler.es_scroll_size, 151 | scroll=self.conf.profiler.es_scroll_time, 152 | body={"query": { 153 | "match": {"base_id": base_id}}}) 154 | 155 | for n in self._hits(response): 156 | trace_id = n["trace_id"] 157 | parent_id = n["parent_id"] 158 | name = n["name"] 159 | project = n["project"] 160 | service = n["service"] 161 | host = n["info"]["host"] 162 | timestamp = n["timestamp"] 163 | 164 | self._append_results(trace_id, parent_id, name, project, service, 165 | host, timestamp, n) 166 | 167 | return self._parse_results() 168 | -------------------------------------------------------------------------------- /osprofiler/drivers/jaeger.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Fujitsu Ltd. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from osprofiler.drivers import base 17 | from osprofiler import exc 18 | 19 | 20 | # TODO(tkajinam): Remove this and the deprecated options after G-release 21 | class Jaeger(base.Driver): 22 | def __init__(self, connection_str, project=None, service=None, host=None, 23 | conf=None, **kwargs): 24 | """Jaeger driver for OSProfiler.""" 25 | 26 | raise exc.CommandError('Jaeger driver is no longer supported') 27 | 28 | @classmethod 29 | def get_name(cls): 30 | return "jaeger" 31 | -------------------------------------------------------------------------------- /osprofiler/drivers/messaging.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import functools 17 | import signal 18 | import time 19 | 20 | from oslo_utils import importutils 21 | 22 | from osprofiler.drivers import base 23 | 24 | 25 | class Messaging(base.Driver): 26 | def __init__(self, connection_str, project=None, service=None, host=None, 27 | context=None, conf=None, transport_url=None, 28 | idle_timeout=1, **kwargs): 29 | """Driver that uses messaging as transport for notifications 30 | 31 | :param connection_str: OSProfiler driver connection string, 32 | equals to messaging:// 33 | :param project: project name that will be included into notification 34 | :param service: service name that will be included into notification 35 | :param host: host name that will be included into notification 36 | :param context: oslo.messaging context 37 | :param conf: oslo.config CONF object 38 | :param transport_url: oslo.messaging transport, e.g. 39 | rabbit://rabbit:password@devstack:5672/ 40 | :param idle_timeout: how long to wait for new notifications after 41 | the last one seen in the trace; this parameter is useful to 42 | collect full trace of asynchronous commands, e.g. when user 43 | runs `osprofiler` right after `openstack server create` 44 | :param kwargs: black hole for any other parameters 45 | """ 46 | 47 | self.oslo_messaging = importutils.try_import("oslo_messaging") 48 | if not self.oslo_messaging: 49 | raise ValueError("Oslo.messaging library is required for " 50 | "messaging driver") 51 | 52 | super().__init__(connection_str, project=project, 53 | service=service, host=host) 54 | 55 | self.context = context 56 | 57 | if not conf: 58 | oslo_config = importutils.try_import("oslo_config") 59 | if not oslo_config: 60 | raise ValueError("Oslo.config library is required for " 61 | "messaging driver") 62 | conf = oslo_config.cfg.CONF 63 | 64 | transport_kwargs = {} 65 | if transport_url: 66 | transport_kwargs["url"] = transport_url 67 | 68 | self.transport = self.oslo_messaging.get_notification_transport( 69 | conf, **transport_kwargs) 70 | self.client = self.oslo_messaging.Notifier( 71 | self.transport, publisher_id=self.host, driver="messaging", 72 | topics=["profiler"], retry=0) 73 | 74 | self.idle_timeout = idle_timeout 75 | 76 | @classmethod 77 | def get_name(cls): 78 | return "messaging" 79 | 80 | def notify(self, info, context=None): 81 | """Send notifications to backend via oslo.messaging notifier API. 82 | 83 | :param info: Contains information about trace element. 84 | In payload dict there are always 3 ids: 85 | "base_id" - uuid that is common for all notifications 86 | related to one trace. 87 | "parent_id" - uuid of parent element in trace 88 | "trace_id" - uuid of current element in trace 89 | With parent_id and trace_id it's quite simple to build 90 | tree of trace elements, which simplify analyze of trace. 91 | 92 | :param context: request context that is mostly used to specify 93 | current active user and tenant. 94 | """ 95 | 96 | info["project"] = self.project 97 | info["service"] = self.service 98 | self.client.info(context or self.context, 99 | "profiler.%s" % info["service"], 100 | info) 101 | 102 | def get_report(self, base_id): 103 | notification_endpoint = NotifyEndpoint(self.oslo_messaging, base_id) 104 | endpoints = [notification_endpoint] 105 | targets = [self.oslo_messaging.Target(topic="profiler")] 106 | server = self.oslo_messaging.notify.get_notification_listener( 107 | self.transport, targets, endpoints, executor="threading") 108 | 109 | state = dict(running=False) 110 | sfn = functools.partial(signal_handler, state=state) 111 | 112 | # modify signal handlers to handle interruption gracefully 113 | old_sigterm_handler = signal.signal(signal.SIGTERM, sfn) 114 | old_sigint_handler = signal.signal(signal.SIGINT, sfn) 115 | 116 | try: 117 | server.start() 118 | except self.oslo_messaging.server.ServerListenError: 119 | # failed to start the server 120 | raise 121 | except SignalExit: 122 | print("Execution interrupted while trying to connect to " 123 | "messaging server. No data was collected.") 124 | return {} 125 | 126 | # connected to server, now read the data 127 | try: 128 | # run until the trace is complete 129 | state["running"] = True 130 | 131 | while state["running"]: 132 | last_read_time = notification_endpoint.get_last_read_time() 133 | wait = self.idle_timeout - (time.time() - last_read_time) 134 | if wait < 0: 135 | state["running"] = False 136 | else: 137 | time.sleep(wait) 138 | except SignalExit: 139 | print("Execution interrupted. Terminating") 140 | finally: 141 | server.stop() 142 | server.wait() 143 | 144 | # restore original signal handlers 145 | signal.signal(signal.SIGTERM, old_sigterm_handler) 146 | signal.signal(signal.SIGINT, old_sigint_handler) 147 | 148 | events = notification_endpoint.get_messages() 149 | 150 | if not events: 151 | print("No events are collected for Trace UUID %s. Please note " 152 | "that osprofiler has read ALL events from profiler topic, " 153 | "but has not found any for specified Trace UUID." % base_id) 154 | 155 | for n in events: 156 | trace_id = n["trace_id"] 157 | parent_id = n["parent_id"] 158 | name = n["name"] 159 | project = n["project"] 160 | service = n["service"] 161 | host = n["info"]["host"] 162 | timestamp = n["timestamp"] 163 | 164 | self._append_results(trace_id, parent_id, name, project, service, 165 | host, timestamp, n) 166 | 167 | return self._parse_results() 168 | 169 | 170 | class NotifyEndpoint: 171 | 172 | def __init__(self, oslo_messaging, base_id): 173 | self.received_messages = [] 174 | self.last_read_time = time.time() 175 | self.filter_rule = oslo_messaging.NotificationFilter( 176 | payload={"base_id": base_id}) 177 | 178 | def info(self, ctxt, publisher_id, event_type, payload, metadata): 179 | self.received_messages.append(payload) 180 | self.last_read_time = time.time() 181 | 182 | def get_messages(self): 183 | return self.received_messages 184 | 185 | def get_last_read_time(self): 186 | return self.last_read_time # time when the latest event was received 187 | 188 | 189 | class SignalExit(BaseException): 190 | pass 191 | 192 | 193 | def signal_handler(signum, frame, state): 194 | state["running"] = False 195 | raise SignalExit() 196 | -------------------------------------------------------------------------------- /osprofiler/drivers/mongodb.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from osprofiler.drivers import base 17 | from osprofiler import exc 18 | 19 | 20 | class MongoDB(base.Driver): 21 | def __init__(self, connection_str, db_name="osprofiler", project=None, 22 | service=None, host=None, **kwargs): 23 | """MongoDB driver for OSProfiler.""" 24 | 25 | super().__init__(connection_str, project=project, 26 | service=service, host=host, **kwargs) 27 | try: 28 | from pymongo import MongoClient 29 | except ImportError: 30 | raise exc.CommandError( 31 | "To use OSProfiler with MongoDB driver, " 32 | "please install `pymongo` library. " 33 | "To install with pip:\n `pip install pymongo`.") 34 | 35 | client = MongoClient(self.connection_str, connect=False) 36 | self.db = client[db_name] 37 | 38 | @classmethod 39 | def get_name(cls): 40 | return "mongodb" 41 | 42 | def notify(self, info): 43 | """Send notifications to MongoDB. 44 | 45 | :param info: Contains information about trace element. 46 | In payload dict there are always 3 ids: 47 | "base_id" - uuid that is common for all notifications 48 | related to one trace. Used to simplify retrieving of all 49 | trace elements from MongoDB. 50 | "parent_id" - uuid of parent element in trace 51 | "trace_id" - uuid of current element in trace 52 | With parent_id and trace_id it's quite simple to build 53 | tree of trace elements, which simplify analyze of trace. 54 | """ 55 | data = info.copy() 56 | data["project"] = self.project 57 | data["service"] = self.service 58 | self.db.profiler.insert_one(data) 59 | 60 | if (self.filter_error_trace 61 | and data.get("info", {}).get("etype") is not None): 62 | self.notify_error_trace(data) 63 | 64 | def notify_error_trace(self, data): 65 | """Store base_id and timestamp of error trace to a separate db.""" 66 | self.db.profiler_error.update( 67 | {"base_id": data["base_id"]}, 68 | {"base_id": data["base_id"], "timestamp": data["timestamp"]}, 69 | upsert=True 70 | ) 71 | 72 | def list_traces(self, fields=None): 73 | """Query all traces from the storage. 74 | 75 | :param fields: Set of trace fields to return. Defaults to 'base_id' 76 | and 'timestamp' 77 | :returns: List of traces, where each trace is a dictionary containing 78 | at least `base_id` and `timestamp`. 79 | """ 80 | fields = set(fields or self.default_trace_fields) 81 | ids = self.db.profiler.find({}).distinct("base_id") 82 | out_format = {"base_id": 1, "timestamp": 1, "_id": 0} 83 | out_format.update({i: 1 for i in fields}) 84 | return [self.db.profiler.find( 85 | {"base_id": i}, out_format).sort("timestamp")[0] for i in ids] 86 | 87 | def list_error_traces(self): 88 | """Returns all traces that have error/exception.""" 89 | out_format = {"base_id": 1, "timestamp": 1, "_id": 0} 90 | return self.db.profiler_error.find({}, out_format) 91 | 92 | def get_report(self, base_id): 93 | """Retrieves and parses notification from MongoDB. 94 | 95 | :param base_id: Base id of trace elements. 96 | """ 97 | for n in self.db.profiler.find({"base_id": base_id}, {"_id": 0}): 98 | trace_id = n["trace_id"] 99 | parent_id = n["parent_id"] 100 | name = n["name"] 101 | project = n["project"] 102 | service = n["service"] 103 | host = n["info"]["host"] 104 | timestamp = n["timestamp"] 105 | 106 | self._append_results(trace_id, parent_id, name, project, service, 107 | host, timestamp, n) 108 | 109 | return self._parse_results() 110 | -------------------------------------------------------------------------------- /osprofiler/drivers/otlp.py: -------------------------------------------------------------------------------- 1 | # All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import collections 16 | from urllib import parse as parser 17 | 18 | from oslo_config import cfg 19 | from oslo_serialization import jsonutils 20 | 21 | from osprofiler import _utils as utils 22 | from osprofiler.drivers import base 23 | from osprofiler import exc 24 | 25 | 26 | class OTLP(base.Driver): 27 | def __init__(self, connection_str, project=None, service=None, host=None, 28 | conf=cfg.CONF, **kwargs): 29 | """OTLP driver using OTLP exporters.""" 30 | 31 | super().__init__(connection_str, project=project, 32 | service=service, host=host, 33 | conf=conf, **kwargs) 34 | try: 35 | from opentelemetry import trace as trace_api 36 | 37 | from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter # noqa 38 | from opentelemetry.sdk.resources import Resource 39 | from opentelemetry.sdk.trace.export import BatchSpanProcessor 40 | from opentelemetry.sdk.trace import TracerProvider 41 | 42 | self.trace_api = trace_api 43 | except ImportError: 44 | raise exc.CommandError( 45 | "To use OSProfiler with OTLP exporters, " 46 | "please install `opentelemetry-sdk` and " 47 | "opentelemetry-exporter-otlp libraries. " 48 | "To install with pip:\n `pip install opentelemetry-sdk " 49 | "opentelemetry-exporter-otlp`.") 50 | 51 | service_name = self._get_service_name(conf, project, service) 52 | resource = Resource(attributes={ 53 | "service.name": service_name 54 | }) 55 | 56 | parsed_url = parser.urlparse(connection_str) 57 | # TODO("sahid"): We also want to handle https scheme? 58 | parsed_url = parsed_url._replace(scheme="http") 59 | 60 | self.trace_api.set_tracer_provider( 61 | TracerProvider(resource=resource)) 62 | self.tracer = self.trace_api.get_tracer(__name__) 63 | 64 | exporter = OTLPSpanExporter("{}/v1/traces".format( 65 | parsed_url.geturl())) 66 | self.trace_api.get_tracer_provider().add_span_processor( 67 | BatchSpanProcessor(exporter)) 68 | 69 | self.spans = collections.deque() 70 | 71 | def _get_service_name(self, conf, project, service): 72 | prefix = conf.profiler_otlp.service_name_prefix 73 | if prefix: 74 | return "{}-{}-{}".format(prefix, project, service) 75 | return "{}-{}".format(project, service) 76 | 77 | @classmethod 78 | def get_name(cls): 79 | return "otlp" 80 | 81 | def _kind(self, name): 82 | if "wsgi" in name: 83 | return self.trace_api.SpanKind.SERVER 84 | elif ("db" in name or "http" in name or "api" in name): 85 | return self.trace_api.SpanKind.CLIENT 86 | return self.trace_api.SpanKind.INTERNAL 87 | 88 | def _name(self, payload): 89 | info = payload["info"] 90 | if info.get("request"): 91 | return "WSGI_{}_{}".format( 92 | info["request"]["method"], info["request"]["path"]) 93 | elif info.get("db"): 94 | return "SQL_{}".format( 95 | info["db"]["statement"].split(' ', 1)[0].upper()) 96 | elif info.get("requests"): 97 | return "REQUESTS_{}_{}".format( 98 | info["requests"]["method"], info["requests"]["hostname"]) 99 | return payload["name"].rstrip("-start") 100 | 101 | def notify(self, payload): 102 | if payload["name"].endswith("start"): 103 | parent = self.trace_api.SpanContext( 104 | trace_id=utils.uuid_to_int128(payload["base_id"]), 105 | span_id=utils.shorten_id(payload["parent_id"]), 106 | is_remote=False, 107 | trace_flags=self.trace_api.TraceFlags( 108 | self.trace_api.TraceFlags.SAMPLED)) 109 | 110 | ctx = self.trace_api.set_span_in_context( 111 | self.trace_api.NonRecordingSpan(parent)) 112 | 113 | # OTLP Tracing span 114 | span = self.tracer.start_span( 115 | name=self._name(payload), 116 | kind=self._kind(payload['name']), 117 | attributes=self.create_span_tags(payload), 118 | context=ctx) 119 | 120 | span._context = self.trace_api.SpanContext( 121 | trace_id=span.context.trace_id, 122 | span_id=utils.shorten_id(payload["trace_id"]), 123 | is_remote=span.context.is_remote, 124 | trace_flags=span.context.trace_flags, 125 | trace_state=span.context.trace_state) 126 | 127 | self.spans.append(span) 128 | else: 129 | span = self.spans.pop() 130 | 131 | # Store result of db call and function call 132 | for call in ("db", "function"): 133 | if payload.get("info", {}).get(call): 134 | span.set_attribute( 135 | "result", payload["info"][call]["result"]) 136 | # Store result of requests 137 | if payload.get("info", {}).get("requests"): 138 | span.set_attribute( 139 | "status_code", payload["info"]["requests"]["status_code"]) 140 | # Span error tag and log 141 | if payload["info"].get("etype"): 142 | span.set_attribute("error", True) 143 | span.add_event("log", { 144 | "error.kind": payload["info"]["etype"], 145 | "message": payload["info"]["message"]}) 146 | span.end() 147 | 148 | def get_report(self, base_id): 149 | return self._parse_results() 150 | 151 | def list_traces(self, fields=None): 152 | return [] 153 | 154 | def list_error_traces(self): 155 | return [] 156 | 157 | def create_span_tags(self, payload): 158 | """Create tags an OpenTracing compatible span. 159 | 160 | :param info: Information from OSProfiler trace. 161 | :returns tags: A dictionary contains standard tags 162 | from OpenTracing sematic conventions, 163 | and some other custom tags related to http, db calls. 164 | """ 165 | tags = {} 166 | info = payload["info"] 167 | 168 | if info.get("db"): 169 | # DB calls 170 | tags["db.statement"] = info["db"]["statement"] 171 | tags["db.params"] = jsonutils.dumps(info["db"]["params"]) 172 | elif info.get("request"): 173 | # WSGI call 174 | tags["http.path"] = info["request"]["path"] 175 | tags["http.query"] = info["request"]["query"] 176 | tags["http.method"] = info["request"]["method"] 177 | tags["http.scheme"] = info["request"]["scheme"] 178 | elif info.get("requests"): 179 | # requests call 180 | tags["http.path"] = info["requests"]["path"] 181 | tags["http.query"] = info["requests"]["query"] 182 | tags["http.method"] = info["requests"]["method"] 183 | tags["http.scheme"] = info["requests"]["scheme"] 184 | tags["http.hostname"] = info["requests"]["hostname"] 185 | tags["http.port"] = info["requests"]["port"] 186 | elif info.get("function"): 187 | # RPC, function calls 188 | if "args" in info["function"]: 189 | tags["args"] = info["function"]["args"] 190 | if "kwargs" in info["function"]: 191 | tags["kwargs"] = info["function"]["kwargs"] 192 | tags["name"] = info["function"]["name"] 193 | 194 | return tags 195 | -------------------------------------------------------------------------------- /osprofiler/drivers/redis_driver.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # Copyright 2016 IBM Corporation. 3 | # All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | from urllib import parse as parser 18 | 19 | from debtcollector import removals 20 | from oslo_config import cfg 21 | from oslo_serialization import jsonutils 22 | 23 | from osprofiler.drivers import base 24 | from osprofiler import exc 25 | 26 | 27 | class Redis(base.Driver): 28 | @removals.removed_kwarg("db", message="'db' parameter is deprecated " 29 | "and will be removed in future. " 30 | "Please specify 'db' in " 31 | "'connection_string' instead.") 32 | def __init__(self, connection_str, db=0, project=None, 33 | service=None, host=None, conf=cfg.CONF, **kwargs): 34 | """Redis driver for OSProfiler.""" 35 | 36 | super().__init__(connection_str, project=project, 37 | service=service, host=host, 38 | conf=conf, **kwargs) 39 | try: 40 | from redis import Redis as _Redis 41 | except ImportError: 42 | raise exc.CommandError( 43 | "To use OSProfiler with Redis driver, " 44 | "please install `redis` library. " 45 | "To install with pip:\n `pip install redis`.") 46 | 47 | # only connection over network is supported with schema 48 | # redis://[:password]@host[:port][/db] 49 | self.db = _Redis.from_url(self.connection_str) 50 | self.namespace_opt = "osprofiler_opt:" 51 | self.namespace = "osprofiler:" # legacy 52 | self.namespace_error = "osprofiler_error:" 53 | 54 | @classmethod 55 | def get_name(cls): 56 | return "redis" 57 | 58 | def notify(self, info): 59 | """Send notifications to Redis. 60 | 61 | :param info: Contains information about trace element. 62 | In payload dict there are always 3 ids: 63 | "base_id" - uuid that is common for all notifications 64 | related to one trace. Used to simplify retrieving of all 65 | trace elements from Redis. 66 | "parent_id" - uuid of parent element in trace 67 | "trace_id" - uuid of current element in trace 68 | With parent_id and trace_id it's quite simple to build 69 | tree of trace elements, which simplify analyze of trace. 70 | """ 71 | data = info.copy() 72 | data["project"] = self.project 73 | data["service"] = self.service 74 | key = self.namespace_opt + data["base_id"] 75 | self.db.lpush(key, jsonutils.dumps(data)) 76 | 77 | if (self.filter_error_trace 78 | and data.get("info", {}).get("etype") is not None): 79 | self.notify_error_trace(data) 80 | 81 | def notify_error_trace(self, data): 82 | """Store base_id and timestamp of error trace to a separate key.""" 83 | key = self.namespace_error + data["base_id"] 84 | value = jsonutils.dumps({ 85 | "base_id": data["base_id"], 86 | "timestamp": data["timestamp"] 87 | }) 88 | self.db.set(key, value) 89 | 90 | def list_traces(self, fields=None): 91 | """Query all traces from the storage. 92 | 93 | :param fields: Set of trace fields to return. Defaults to 'base_id' 94 | and 'timestamp' 95 | :returns: List of traces, where each trace is a dictionary containing 96 | at least `base_id` and `timestamp`. 97 | """ 98 | fields = set(fields or self.default_trace_fields) 99 | 100 | # first get legacy events 101 | result = self._list_traces_legacy(fields) 102 | 103 | # with optimized schema trace events are stored in a list 104 | ids = self.db.scan_iter(match=self.namespace_opt + "*") 105 | for i in ids: 106 | # for each trace query the first event to have a timestamp 107 | first_event = jsonutils.loads(self.db.lindex(i, 1)) 108 | result.append({key: value for key, value in first_event.items() 109 | if key in fields}) 110 | return result 111 | 112 | def _list_traces_legacy(self, fields): 113 | # With current schema every event is stored under its own unique key 114 | # To query all traces we first need to get all keys, then 115 | # get all events, sort them and pick up only the first one 116 | ids = self.db.scan_iter(match=self.namespace + "*") 117 | traces = [jsonutils.loads(self.db.get(i)) for i in ids] 118 | traces.sort(key=lambda x: x["timestamp"]) 119 | seen_ids = set() 120 | result = [] 121 | for trace in traces: 122 | if trace["base_id"] not in seen_ids: 123 | seen_ids.add(trace["base_id"]) 124 | result.append({key: value for key, value in trace.items() 125 | if key in fields}) 126 | return result 127 | 128 | def list_error_traces(self): 129 | """Returns all traces that have error/exception.""" 130 | ids = self.db.scan_iter(match=self.namespace_error + "*") 131 | traces = [jsonutils.loads(self.db.get(i)) for i in ids] 132 | traces.sort(key=lambda x: x["timestamp"]) 133 | seen_ids = set() 134 | result = [] 135 | for trace in traces: 136 | if trace["base_id"] not in seen_ids: 137 | seen_ids.add(trace["base_id"]) 138 | result.append(trace) 139 | 140 | return result 141 | 142 | def get_report(self, base_id): 143 | """Retrieves and parses notification from Redis. 144 | 145 | :param base_id: Base id of trace elements. 146 | """ 147 | def iterate_events(): 148 | for key in self.db.scan_iter( 149 | match=self.namespace + base_id + "*"): # legacy 150 | yield self.db.get(key) 151 | 152 | yield from self.db.lrange(self.namespace_opt + base_id, 0, -1) 153 | 154 | for data in iterate_events(): 155 | n = jsonutils.loads(data) 156 | trace_id = n["trace_id"] 157 | parent_id = n["parent_id"] 158 | name = n["name"] 159 | project = n["project"] 160 | service = n["service"] 161 | host = n["info"]["host"] 162 | timestamp = n["timestamp"] 163 | 164 | self._append_results(trace_id, parent_id, name, project, service, 165 | host, timestamp, n) 166 | 167 | return self._parse_results() 168 | 169 | 170 | class RedisSentinel(Redis, base.Driver): 171 | @removals.removed_kwarg("db", message="'db' parameter is deprecated " 172 | "and will be removed in future. " 173 | "Please specify 'db' in " 174 | "'connection_string' instead.") 175 | def __init__(self, connection_str, db=0, project=None, 176 | service=None, host=None, conf=cfg.CONF, **kwargs): 177 | """Redis driver for OSProfiler.""" 178 | 179 | super().__init__(connection_str, project=project, 180 | service=service, host=host, 181 | conf=conf, **kwargs) 182 | try: 183 | from redis.sentinel import Sentinel 184 | except ImportError: 185 | raise exc.CommandError( 186 | "To use this command, you should install " 187 | "'redis' manually. Use command:\n " 188 | "'pip install redis'.") 189 | 190 | self.conf = conf 191 | socket_timeout = self.conf.profiler.socket_timeout 192 | parsed_url = parser.urlparse(self.connection_str) 193 | sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))], 194 | password=parsed_url.password, 195 | socket_timeout=socket_timeout) 196 | self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name, 197 | socket_timeout=socket_timeout) 198 | 199 | @classmethod 200 | def get_name(cls): 201 | return "redissentinel" 202 | -------------------------------------------------------------------------------- /osprofiler/drivers/sqlalchemy_driver.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 SUSE Linux GmbH 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import logging 17 | 18 | from oslo_serialization import jsonutils 19 | 20 | from osprofiler.drivers import base 21 | from osprofiler import exc 22 | 23 | LOG = logging.getLogger(__name__) 24 | 25 | 26 | class SQLAlchemyDriver(base.Driver): 27 | def __init__(self, connection_str, project=None, service=None, host=None, 28 | **kwargs): 29 | super().__init__(connection_str, project=project, 30 | service=service, host=host) 31 | 32 | try: 33 | from sqlalchemy import create_engine 34 | from sqlalchemy import Table, MetaData, Column 35 | from sqlalchemy import String, JSON, Integer 36 | except ImportError: 37 | LOG.exception("To use this command, install 'SQLAlchemy'") 38 | else: 39 | self._metadata = MetaData() 40 | self._data_table = Table( 41 | "data", self._metadata, 42 | Column("id", Integer, primary_key=True), 43 | # timestamp - date/time of the trace point 44 | Column("timestamp", String(26), index=True), 45 | # base_id - uuid common for all notifications related to 46 | # one trace 47 | Column("base_id", String(255), index=True), 48 | # parent_id - uuid of parent element in trace 49 | Column("parent_id", String(255), index=True), 50 | # trace_id - uuid of current element in trace 51 | Column("trace_id", String(255), index=True), 52 | Column("project", String(255), index=True), 53 | Column("host", String(255), index=True), 54 | Column("service", String(255), index=True), 55 | # name - trace point name 56 | Column("name", String(255), index=True), 57 | Column("data", JSON) 58 | ) 59 | 60 | # we don't want to kill any service that does use osprofiler 61 | try: 62 | self._engine = create_engine(connection_str) 63 | self._conn = self._engine.connect() 64 | 65 | # FIXME(toabctl): Not the best idea to create the table on every 66 | # startup when using the sqlalchemy driver... 67 | self._metadata.create_all(self._engine, checkfirst=True) 68 | except Exception: 69 | LOG.exception("Failed to create engine/connection and setup " 70 | "intial database tables") 71 | 72 | @classmethod 73 | def get_name(cls): 74 | return "sqlalchemy" 75 | 76 | def notify(self, info, context=None): 77 | """Write a notification the the database""" 78 | data = info.copy() 79 | base_id = data.pop("base_id", None) 80 | timestamp = data.pop("timestamp", None) 81 | parent_id = data.pop("parent_id", None) 82 | trace_id = data.pop("trace_id", None) 83 | project = data.pop("project", self.project) 84 | host = data.pop("host", self.host) 85 | service = data.pop("service", self.service) 86 | name = data.pop("name", None) 87 | 88 | try: 89 | ins = self._data_table.insert().values( 90 | timestamp=timestamp, 91 | base_id=base_id, 92 | parent_id=parent_id, 93 | trace_id=trace_id, 94 | project=project, 95 | service=service, 96 | host=host, 97 | name=name, 98 | data=jsonutils.dumps(data) 99 | ) 100 | self._conn.execute(ins) 101 | except Exception: 102 | LOG.exception("Can not store osprofiler tracepoint {} " 103 | "(base_id {})".format(trace_id, base_id)) 104 | 105 | def list_traces(self, fields=None): 106 | try: 107 | from sqlalchemy.sql import select 108 | except ImportError: 109 | raise exc.CommandError( 110 | "To use this command, you should install 'SQLAlchemy'") 111 | stmt = select([self._data_table]) 112 | seen_ids = set() 113 | result = [] 114 | traces = self._conn.execute(stmt).fetchall() 115 | for trace in traces: 116 | if trace["base_id"] not in seen_ids: 117 | seen_ids.add(trace["base_id"]) 118 | result.append({key: value for key, value in trace.items() 119 | if key in fields}) 120 | return result 121 | 122 | def get_report(self, base_id): 123 | try: 124 | from sqlalchemy.sql import select 125 | except ImportError: 126 | raise exc.CommandError( 127 | "To use this command, you should install 'SQLAlchemy'") 128 | stmt = select([self._data_table]).where( 129 | self._data_table.c.base_id == base_id) 130 | results = self._conn.execute(stmt).fetchall() 131 | for n in results: 132 | timestamp = n["timestamp"] 133 | trace_id = n["trace_id"] 134 | parent_id = n["parent_id"] 135 | name = n["name"] 136 | project = n["project"] 137 | service = n["service"] 138 | host = n["host"] 139 | data = jsonutils.loads(n["data"]) 140 | self._append_results(trace_id, parent_id, name, project, service, 141 | host, timestamp, data) 142 | return self._parse_results() 143 | -------------------------------------------------------------------------------- /osprofiler/exc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | 17 | class CommandError(Exception): 18 | """Invalid usage of CLI.""" 19 | 20 | def __init__(self, message=None): 21 | self.message = message 22 | 23 | def __str__(self): 24 | return self.message or self.__class__.__doc__ 25 | 26 | 27 | class LogInsightAPIError(Exception): 28 | pass 29 | 30 | 31 | class LogInsightLoginTimeout(Exception): 32 | pass 33 | -------------------------------------------------------------------------------- /osprofiler/hacking/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/hacking/__init__.py -------------------------------------------------------------------------------- /osprofiler/initializer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from osprofiler import notifier 17 | from osprofiler import requests 18 | from osprofiler import web 19 | 20 | 21 | def init_from_conf(conf, context, project, service, host, **kwargs): 22 | """Initialize notifier from service configuration 23 | 24 | :param conf: service configuration 25 | :param context: request context 26 | :param project: project name (keystone, cinder etc.) 27 | :param service: service name that will be profiled 28 | :param host: hostname or host IP address that the service will be 29 | running on. 30 | :param kwargs: other arguments for notifier creation 31 | """ 32 | connection_str = conf.profiler.connection_string 33 | _notifier = notifier.create( 34 | connection_str, 35 | context=context, 36 | project=project, 37 | service=service, 38 | host=host, 39 | conf=conf, 40 | **kwargs) 41 | notifier.set(_notifier) 42 | web.enable(conf.profiler.hmac_keys) 43 | if conf.profiler.trace_requests: 44 | requests.enable() 45 | -------------------------------------------------------------------------------- /osprofiler/notifier.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import logging 17 | 18 | from osprofiler.drivers import base 19 | 20 | 21 | LOG = logging.getLogger(__name__) 22 | 23 | 24 | def _noop_notifier(info, context=None): 25 | """Do nothing on notify().""" 26 | 27 | 28 | # NOTE(boris-42): By default we are using noop notifier. 29 | __notifier = _noop_notifier 30 | __notifier_cache = {} # map: connection-string -> notifier 31 | 32 | 33 | def notify(info): 34 | """Passes the profiling info to the notifier callable. 35 | 36 | :param info: dictionary with profiling information 37 | """ 38 | __notifier(info) 39 | 40 | 41 | def get(): 42 | """Returns notifier callable.""" 43 | return __notifier 44 | 45 | 46 | def set(notifier): 47 | """Service that are going to use profiler should set callable notifier. 48 | 49 | Callable notifier is instance of callable object, that accept exactly 50 | one argument "info". "info" - is dictionary of values that contains 51 | profiling information. 52 | """ 53 | global __notifier 54 | __notifier = notifier 55 | 56 | 57 | def create(connection_string, *args, **kwargs): 58 | """Create notifier based on specified plugin_name 59 | 60 | :param connection_string: connection string which specifies the storage 61 | driver for notifier 62 | :param args: args that will be passed to the driver's __init__ method 63 | :param kwargs: kwargs that will be passed to the driver's __init__ method 64 | :returns: Callable notifier method 65 | """ 66 | global __notifier_cache 67 | if connection_string not in __notifier_cache: 68 | try: 69 | driver = base.get_driver(connection_string, *args, **kwargs) 70 | __notifier_cache[connection_string] = driver.notify 71 | LOG.info("osprofiler is enabled with connection string: %s", 72 | connection_string) 73 | except Exception: 74 | LOG.exception("Could not initialize driver for connection string " 75 | "%s, osprofiler is disabled", connection_string) 76 | __notifier_cache[connection_string] = _noop_notifier 77 | 78 | return __notifier_cache[connection_string] 79 | 80 | 81 | def clear_notifier_cache(): 82 | __notifier_cache.clear() 83 | -------------------------------------------------------------------------------- /osprofiler/requests.py: -------------------------------------------------------------------------------- 1 | # All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import logging as log 16 | from urllib import parse as parser 17 | 18 | from osprofiler import profiler 19 | from osprofiler import web 20 | 21 | 22 | # Register an OSProfiler HTTP Adapter that will profile any call made with 23 | # requests. 24 | 25 | LOG = log.getLogger(__name__) 26 | 27 | _FUNC = None 28 | 29 | try: 30 | from requests.adapters import HTTPAdapter 31 | except ImportError: 32 | pass 33 | else: 34 | def send(self, request, *args, **kwargs): 35 | parsed_url = parser.urlparse(request.url) 36 | 37 | # Best effort guessing port if needed 38 | port = parsed_url.port or "" 39 | if not port and parsed_url.scheme == "http": 40 | port = 80 41 | elif not port and parsed_url.scheme == "https": 42 | port = 443 43 | 44 | profiler.start(parsed_url.scheme, info={"requests": { 45 | "method": request.method, 46 | "query": parsed_url.query, 47 | "path": parsed_url.path, 48 | "hostname": parsed_url.hostname, 49 | "port": port, 50 | "scheme": parsed_url.scheme}}) 51 | 52 | # Profiling headers are overrident to take in account this new 53 | # context/span. 54 | request.headers.update( 55 | web.get_trace_id_headers()) 56 | 57 | response = _FUNC(self, request, *args, **kwargs) 58 | 59 | profiler.stop(info={"requests": { 60 | "status_code": response.status_code}}) 61 | 62 | return response 63 | 64 | _FUNC = HTTPAdapter.send 65 | 66 | 67 | def enable(): 68 | if _FUNC: 69 | HTTPAdapter.send = send 70 | LOG.debug("profiling requests enabled") 71 | else: 72 | LOG.warning("unable to activate profiling for requests, " 73 | "please ensure that python requests is installed.") 74 | -------------------------------------------------------------------------------- /osprofiler/sqlalchemy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import contextlib 17 | import logging as log 18 | 19 | from oslo_utils import reflection 20 | 21 | from osprofiler import profiler 22 | 23 | LOG = log.getLogger(__name__) 24 | 25 | _DISABLED = False 26 | 27 | 28 | def disable(): 29 | """Disable tracing of all DB queries. Reduce a lot size of profiles.""" 30 | global _DISABLED 31 | _DISABLED = True 32 | 33 | 34 | def enable(): 35 | """add_tracing adds event listeners for sqlalchemy.""" 36 | 37 | global _DISABLED 38 | _DISABLED = False 39 | 40 | 41 | def add_tracing(sqlalchemy, engine, name, hide_result=True): 42 | """Add tracing to all sqlalchemy calls.""" 43 | 44 | if not _DISABLED: 45 | sqlalchemy.event.listen(engine, "before_cursor_execute", 46 | _before_cursor_execute(name)) 47 | sqlalchemy.event.listen( 48 | engine, "after_cursor_execute", 49 | _after_cursor_execute(hide_result=hide_result) 50 | ) 51 | sqlalchemy.event.listen(engine, "handle_error", handle_error) 52 | 53 | 54 | @contextlib.contextmanager 55 | def wrap_session(sqlalchemy, sess): 56 | with sess as s: 57 | if not getattr(s.bind, "traced", False): 58 | add_tracing(sqlalchemy, s.bind, "db") 59 | s.bind.traced = True 60 | yield s 61 | 62 | 63 | def _before_cursor_execute(name): 64 | """Add listener that will send trace info before query is executed.""" 65 | 66 | def handler(conn, cursor, statement, params, context, executemany): 67 | info = {"db": { 68 | "statement": statement, 69 | "params": params} 70 | } 71 | profiler.start(name, info=info) 72 | 73 | return handler 74 | 75 | 76 | def _after_cursor_execute(hide_result=True): 77 | """Add listener that will send trace info after query is executed. 78 | 79 | :param hide_result: Boolean value to hide or show SQL result in trace. 80 | True - hide SQL result (default). 81 | False - show SQL result in trace. 82 | """ 83 | 84 | def handler(conn, cursor, statement, params, context, executemany): 85 | if not hide_result: 86 | # Add SQL result to trace info in *-stop phase 87 | info = { 88 | "db": { 89 | "result": str(cursor._rows) 90 | } 91 | } 92 | profiler.stop(info=info) 93 | else: 94 | profiler.stop() 95 | 96 | return handler 97 | 98 | 99 | def handle_error(exception_context): 100 | """Handle SQLAlchemy errors""" 101 | exception_class_name = reflection.get_class_name( 102 | exception_context.original_exception) 103 | original_exception = str(exception_context.original_exception) 104 | chained_exception = str(exception_context.chained_exception) 105 | 106 | info = { 107 | "etype": exception_class_name, 108 | "message": original_exception, 109 | "db": { 110 | "original_exception": original_exception, 111 | "chained_exception": chained_exception 112 | } 113 | } 114 | profiler.stop(info=info) 115 | LOG.debug("OSProfiler has handled SQLAlchemy error: %s", 116 | original_exception) 117 | -------------------------------------------------------------------------------- /osprofiler/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/tests/__init__.py -------------------------------------------------------------------------------- /osprofiler/tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/tests/functional/__init__.py -------------------------------------------------------------------------------- /osprofiler/tests/functional/config.cfg: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | transport_url=rabbit://localhost:5672/ 3 | 4 | [profiler] 5 | connection_string="messaging://" 6 | -------------------------------------------------------------------------------- /osprofiler/tests/functional/test_driver.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 VMware, Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import logging 17 | import os 18 | 19 | from oslo_config import cfg 20 | 21 | from osprofiler.drivers import base 22 | from osprofiler import initializer 23 | from osprofiler import opts 24 | from osprofiler import profiler 25 | from osprofiler.tests import test 26 | 27 | 28 | CONF = cfg.CONF 29 | LOG = logging.getLogger(__name__) 30 | 31 | 32 | @profiler.trace_cls("rpc", hide_args=True) 33 | class Foo: 34 | def bar(self, x): 35 | return self.baz(x, x) 36 | 37 | def baz(self, x, y): 38 | return x * y 39 | 40 | 41 | class DriverTestCase(test.FunctionalTestCase): 42 | 43 | SERVICE = "service" 44 | PROJECT = "project" 45 | 46 | def setUp(self): 47 | super().setUp() 48 | CONF(["--config-file", os.path.dirname(__file__) + "/config.cfg"]) 49 | opts.set_defaults(CONF, 50 | enabled=True, 51 | trace_sqlalchemy=False, 52 | hmac_keys="SECRET_KEY") 53 | 54 | def _assert_dict(self, info, **kwargs): 55 | for key in kwargs: 56 | self.assertEqual(kwargs[key], info[key]) 57 | 58 | def _assert_child_dict(self, child, base_id, parent_id, name, fn_name): 59 | self.assertEqual(parent_id, child["parent_id"]) 60 | 61 | exp_info = {"name": "rpc", 62 | "service": self.SERVICE, 63 | "project": self.PROJECT} 64 | self._assert_dict(child["info"], **exp_info) 65 | 66 | raw_start = child["info"]["meta.raw_payload.%s-start" % name] 67 | self.assertEqual(fn_name, raw_start["info"]["function"]["name"]) 68 | exp_raw = {"name": "%s-start" % name, 69 | "service": self.SERVICE, 70 | "trace_id": child["trace_id"], 71 | "project": self.PROJECT, 72 | "base_id": base_id} 73 | self._assert_dict(raw_start, **exp_raw) 74 | 75 | raw_stop = child["info"]["meta.raw_payload.%s-stop" % name] 76 | exp_raw["name"] = "%s-stop" % name 77 | self._assert_dict(raw_stop, **exp_raw) 78 | 79 | def test_get_report(self): 80 | # initialize profiler notifier (the same way as in services) 81 | initializer.init_from_conf( 82 | CONF, {}, self.PROJECT, self.SERVICE, "host") 83 | profiler.init("SECRET_KEY") 84 | 85 | # grab base_id 86 | base_id = profiler.get().get_base_id() 87 | 88 | # execute profiled code 89 | foo = Foo() 90 | foo.bar(1) 91 | 92 | # instantiate report engine (the same way as in osprofiler CLI) 93 | engine = base.get_driver(CONF.profiler.connection_string, 94 | project=self.PROJECT, 95 | service=self.SERVICE, 96 | host="host", 97 | conf=CONF) 98 | 99 | # generate the report 100 | report = engine.get_report(base_id) 101 | LOG.debug("OSProfiler report: %s", report) 102 | 103 | # verify the report 104 | self.assertEqual("total", report["info"]["name"]) 105 | self.assertEqual(2, report["stats"]["rpc"]["count"]) 106 | self.assertEqual(1, len(report["children"])) 107 | 108 | cbar = report["children"][0] 109 | self._assert_child_dict( 110 | cbar, base_id, base_id, "rpc", 111 | "osprofiler.tests.functional.test_driver.Foo.bar") 112 | 113 | self.assertEqual(1, len(cbar["children"])) 114 | cbaz = cbar["children"][0] 115 | self._assert_child_dict( 116 | cbaz, base_id, cbar["trace_id"], "rpc", 117 | "osprofiler.tests.functional.test_driver.Foo.baz") 118 | 119 | 120 | class RedisDriverTestCase(DriverTestCase): 121 | def setUp(self): 122 | super(DriverTestCase, self).setUp() 123 | CONF([]) 124 | opts.set_defaults(CONF, 125 | connection_string="redis://localhost:6379", 126 | enabled=True, 127 | trace_sqlalchemy=False, 128 | hmac_keys="SECRET_KEY") 129 | 130 | def test_list_traces(self): 131 | # initialize profiler notifier (the same way as in services) 132 | initializer.init_from_conf( 133 | CONF, {}, self.PROJECT, self.SERVICE, "host") 134 | profiler.init("SECRET_KEY") 135 | 136 | # grab base_id 137 | base_id = profiler.get().get_base_id() 138 | 139 | # execute profiled code 140 | foo = Foo() 141 | foo.bar(1) 142 | 143 | # instantiate report engine (the same way as in osprofiler CLI) 144 | engine = base.get_driver(CONF.profiler.connection_string, 145 | project=self.PROJECT, 146 | service=self.SERVICE, 147 | host="host", 148 | conf=CONF) 149 | 150 | # generate the report 151 | traces = engine.list_traces() 152 | LOG.debug("Collected traces: %s", traces) 153 | 154 | # ensure trace with base_id is in the list of traces 155 | self.assertIn(base_id, [t["base_id"] for t in traces]) 156 | -------------------------------------------------------------------------------- /osprofiler/tests/test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import logging 17 | import sys 18 | 19 | from testtools import testcase 20 | 21 | 22 | class TestCase(testcase.TestCase): 23 | """Test case base class for all osprofiler unit tests.""" 24 | pass 25 | 26 | 27 | class FunctionalTestCase(TestCase): 28 | """Base for functional tests""" 29 | 30 | def setUp(self): 31 | super().setUp() 32 | 33 | logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) 34 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/tests/unit/__init__.py -------------------------------------------------------------------------------- /osprofiler/tests/unit/cmd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/tests/unit/cmd/__init__.py -------------------------------------------------------------------------------- /osprofiler/tests/unit/cmd/test_shell.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import io 17 | import json 18 | import os 19 | import sys 20 | from unittest import mock 21 | 22 | import ddt 23 | 24 | from osprofiler.cmd import shell 25 | from osprofiler import exc 26 | from osprofiler.tests import test 27 | 28 | 29 | @ddt.ddt 30 | class ShellTestCase(test.TestCase): 31 | 32 | TRACE_ID = "c598094d-bbee-40b6-b317-d76003b679d3" 33 | 34 | def setUp(self): 35 | super().setUp() 36 | self.old_environment = os.environ.copy() 37 | 38 | def tearDown(self): 39 | super().tearDown() 40 | os.environ = self.old_environment 41 | 42 | def _trace_show_cmd(self, format_=None): 43 | cmd = "trace show --connection-string redis:// %s" % self.TRACE_ID 44 | return cmd if format_ is None else "{} --{}".format(cmd, format_) 45 | 46 | @mock.patch("sys.stdout", io.StringIO()) 47 | @mock.patch("osprofiler.cmd.shell.OSProfilerShell") 48 | def test_shell_main(self, mock_shell): 49 | mock_shell.side_effect = exc.CommandError("some_message") 50 | shell.main() 51 | self.assertEqual("some_message\n", sys.stdout.getvalue()) 52 | 53 | def run_command(self, cmd): 54 | shell.OSProfilerShell(cmd.split()) 55 | 56 | def _test_with_command_error(self, cmd, expected_message): 57 | try: 58 | self.run_command(cmd) 59 | except exc.CommandError as actual_error: 60 | self.assertEqual(str(actual_error), expected_message) 61 | else: 62 | raise ValueError( 63 | "Expected: `osprofiler.exc.CommandError` is raised with " 64 | "message: '%s'." % expected_message) 65 | 66 | @mock.patch("osprofiler.drivers.redis_driver.Redis.get_report") 67 | def test_trace_show_no_selected_format(self, mock_get): 68 | mock_get.return_value = self._create_mock_notifications() 69 | msg = ("You should choose one of the following output formats: " 70 | "json, html or dot.") 71 | self._test_with_command_error(self._trace_show_cmd(), msg) 72 | 73 | @mock.patch("osprofiler.drivers.redis_driver.Redis.get_report") 74 | @ddt.data(None, {"info": {"started": 0, "finished": 1, "name": "total"}, 75 | "children": []}) 76 | def test_trace_show_trace_id_not_found(self, notifications, mock_get): 77 | mock_get.return_value = notifications 78 | 79 | msg = ("Trace with UUID %s not found. Please check the HMAC key " 80 | "used in the command." % self.TRACE_ID) 81 | 82 | self._test_with_command_error(self._trace_show_cmd(), msg) 83 | 84 | def _create_mock_notifications(self): 85 | notifications = { 86 | "info": { 87 | "started": 0, 88 | "finished": 1, 89 | "name": "total" 90 | }, 91 | "children": [{ 92 | "info": { 93 | "started": 0, 94 | "finished": 1, 95 | "name": "total" 96 | }, 97 | "children": [] 98 | }] 99 | } 100 | return notifications 101 | 102 | @mock.patch("sys.stdout", io.StringIO()) 103 | @mock.patch("osprofiler.drivers.redis_driver.Redis.get_report") 104 | def test_trace_show_in_json(self, mock_get): 105 | notifications = self._create_mock_notifications() 106 | mock_get.return_value = notifications 107 | 108 | self.run_command(self._trace_show_cmd(format_="json")) 109 | self.assertEqual("%s\n" % json.dumps(notifications, indent=2, 110 | separators=(",", ": "),), 111 | sys.stdout.getvalue()) 112 | 113 | @mock.patch("sys.stdout", io.StringIO()) 114 | @mock.patch("osprofiler.drivers.redis_driver.Redis.get_report") 115 | def test_trace_show_in_html(self, mock_get): 116 | notifications = self._create_mock_notifications() 117 | mock_get.return_value = notifications 118 | 119 | # NOTE(akurilin): to simplify assert statement, html-template should be 120 | # replaced. 121 | html_template = ( 122 | "A long time ago in a galaxy far, far away..." 123 | " some_data = $DATA" 124 | "It is a period of civil war. Rebel" 125 | "spaceships, striking from a hidden" 126 | "base, have won their first victory" 127 | "against the evil Galactic Empire.") 128 | 129 | with mock.patch("osprofiler.cmd.commands.open", 130 | mock.mock_open(read_data=html_template), create=True): 131 | self.run_command(self._trace_show_cmd(format_="html")) 132 | self.assertEqual("A long time ago in a galaxy far, far away..." 133 | " some_data = %s" 134 | "It is a period of civil war. Rebel" 135 | "spaceships, striking from a hidden" 136 | "base, have won their first victory" 137 | "against the evil Galactic Empire." 138 | "\n" % json.dumps(notifications, indent=4, 139 | separators=(",", ": ")), 140 | sys.stdout.getvalue()) 141 | 142 | @mock.patch("sys.stdout", io.StringIO()) 143 | @mock.patch("osprofiler.drivers.redis_driver.Redis.get_report") 144 | def test_trace_show_write_to_file(self, mock_get): 145 | notifications = self._create_mock_notifications() 146 | mock_get.return_value = notifications 147 | 148 | with mock.patch("osprofiler.cmd.commands.open", 149 | mock.mock_open(), create=True) as mock_open: 150 | self.run_command("%s --out='/file'" % 151 | self._trace_show_cmd(format_="json")) 152 | 153 | output = mock_open.return_value.__enter__.return_value 154 | output.write.assert_called_once_with( 155 | json.dumps(notifications, indent=2, separators=(",", ": "))) 156 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/doc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/tests/unit/doc/__init__.py -------------------------------------------------------------------------------- /osprofiler/tests/unit/doc/test_specs.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import glob 14 | import os 15 | import re 16 | 17 | import docutils.core 18 | 19 | from osprofiler.tests import test 20 | 21 | 22 | class TitlesTestCase(test.TestCase): 23 | 24 | specs_path = os.path.join( 25 | os.path.dirname(__file__), 26 | os.pardir, os.pardir, os.pardir, os.pardir, 27 | "doc", "specs") 28 | 29 | def _get_title(self, section_tree): 30 | section = {"subtitles": []} 31 | for node in section_tree: 32 | if node.tagname == "title": 33 | section["name"] = node.rawsource 34 | elif node.tagname == "section": 35 | subsection = self._get_title(node) 36 | section["subtitles"].append(subsection["name"]) 37 | return section 38 | 39 | def _get_titles(self, spec): 40 | titles = {} 41 | for node in spec: 42 | if node.tagname == "section": 43 | # Note subsection subtitles are thrown away 44 | section = self._get_title(node) 45 | titles[section["name"]] = section["subtitles"] 46 | return titles 47 | 48 | def _check_titles(self, filename, expect, actual): 49 | missing_sections = [x for x in expect.keys() if x not in actual.keys()] 50 | extra_sections = [x for x in actual.keys() if x not in expect.keys()] 51 | 52 | msgs = [] 53 | if len(missing_sections) > 0: 54 | msgs.append("Missing sections: %s" % missing_sections) 55 | if len(extra_sections) > 0: 56 | msgs.append("Extra sections: %s" % extra_sections) 57 | 58 | for section in expect.keys(): 59 | missing_subsections = [x for x in expect[section] 60 | if x not in actual.get(section, {})] 61 | # extra subsections are allowed 62 | if len(missing_subsections) > 0: 63 | msgs.append("Section '%s' is missing subsections: %s" 64 | % (section, missing_subsections)) 65 | 66 | if len(msgs) > 0: 67 | self.fail("While checking '%s':\n %s" 68 | % (filename, "\n ".join(msgs))) 69 | 70 | def _check_lines_wrapping(self, tpl, raw): 71 | for i, line in enumerate(raw.split("\n")): 72 | if "http://" in line or "https://" in line: 73 | continue 74 | self.assertTrue( 75 | len(line) < 80, 76 | msg="%s:%d: Line limited to a maximum of 79 characters." % 77 | (tpl, i + 1)) 78 | 79 | def _check_no_cr(self, tpl, raw): 80 | matches = re.findall("\r", raw) 81 | self.assertEqual( 82 | len(matches), 0, 83 | "Found %s literal carriage returns in file %s" % 84 | (len(matches), tpl)) 85 | 86 | def _check_trailing_spaces(self, tpl, raw): 87 | for i, line in enumerate(raw.split("\n")): 88 | trailing_spaces = re.findall(" +$", line) 89 | self.assertEqual( 90 | len(trailing_spaces), 0, 91 | "Found trailing spaces on line {} of {}".format(i + 1, tpl)) 92 | 93 | def test_template(self): 94 | with open(os.path.join(self.specs_path, "template.rst")) as f: 95 | template = f.read() 96 | 97 | spec = docutils.core.publish_doctree(template) 98 | template_titles = self._get_titles(spec) 99 | 100 | for d in ["implemented", "in-progress"]: 101 | spec_dir = "{}/{}".format(self.specs_path, d) 102 | 103 | self.assertTrue(os.path.isdir(spec_dir), 104 | "%s is not a directory" % spec_dir) 105 | for filename in glob.glob(spec_dir + "/*"): 106 | if filename.endswith("README.rst"): 107 | continue 108 | 109 | self.assertTrue( 110 | filename.endswith(".rst"), 111 | "spec's file must have .rst ext. Found: %s" % filename) 112 | with open(filename) as f: 113 | data = f.read() 114 | 115 | titles = self._get_titles(docutils.core.publish_doctree(data)) 116 | self._check_titles(filename, template_titles, titles) 117 | self._check_lines_wrapping(filename, data) 118 | self._check_no_cr(filename, data) 119 | self._check_trailing_spaces(filename, data) 120 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/drivers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/osprofiler/tests/unit/drivers/__init__.py -------------------------------------------------------------------------------- /osprofiler/tests/unit/drivers/test_base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from osprofiler.drivers import base 17 | from osprofiler.tests import test 18 | 19 | 20 | class NotifierBaseTestCase(test.TestCase): 21 | 22 | def test_factory(self): 23 | 24 | class A(base.Driver): 25 | @classmethod 26 | def get_name(cls): 27 | return "a" 28 | 29 | def notify(self, a): 30 | return a 31 | 32 | self.assertEqual(10, base.get_driver("a://").notify(10)) 33 | 34 | def test_factory_with_args(self): 35 | 36 | class B(base.Driver): 37 | 38 | def __init__(self, c_str, a, b=10): 39 | self.a = a 40 | self.b = b 41 | 42 | @classmethod 43 | def get_name(cls): 44 | return "b" 45 | 46 | def notify(self, c): 47 | return self.a + self.b + c 48 | 49 | self.assertEqual(22, base.get_driver("b://", 5, b=7).notify(10)) 50 | 51 | def test_driver_not_found(self): 52 | self.assertRaises(ValueError, base.get_driver, 53 | "Driver not found for connection string: " 54 | "nonexisting://") 55 | 56 | def test_build_empty_tree(self): 57 | class C(base.Driver): 58 | @classmethod 59 | def get_name(cls): 60 | return "c" 61 | 62 | self.assertEqual([], base.get_driver("c://")._build_tree({})) 63 | 64 | def test_build_complex_tree(self): 65 | class D(base.Driver): 66 | @classmethod 67 | def get_name(cls): 68 | return "d" 69 | 70 | test_input = { 71 | "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}}, 72 | "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}}, 73 | "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}}, 74 | "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}}, 75 | "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}}, 76 | "113": {"parent_id": "11", "trace_id": "113", 77 | "info": {"started": 3}}, 78 | "112": {"parent_id": "11", "trace_id": "112", 79 | "info": {"started": 2}}, 80 | "114": {"parent_id": "11", "trace_id": "114", 81 | "info": {"started": 5}} 82 | } 83 | 84 | expected_output = [ 85 | { 86 | "parent_id": "0", 87 | "trace_id": "1", 88 | "info": {"started": 0}, 89 | "children": [ 90 | { 91 | "parent_id": "1", 92 | "trace_id": "11", 93 | "info": {"started": 1}, 94 | "children": [ 95 | {"parent_id": "11", "trace_id": "112", 96 | "info": {"started": 2}, "children": []}, 97 | {"parent_id": "11", "trace_id": "113", 98 | "info": {"started": 3}, "children": []}, 99 | {"parent_id": "11", "trace_id": "114", 100 | "info": {"started": 5}, "children": []} 101 | ] 102 | } 103 | ] 104 | }, 105 | { 106 | "parent_id": "0", 107 | "trace_id": "2", 108 | "info": {"started": 1}, 109 | "children": [ 110 | {"parent_id": "2", "trace_id": "21", 111 | "info": {"started": 6}, "children": []}, 112 | {"parent_id": "2", "trace_id": "22", 113 | "info": {"started": 7}, "children": []} 114 | ] 115 | } 116 | ] 117 | 118 | self.assertEqual( 119 | expected_output, base.get_driver("d://")._build_tree(test_input)) 120 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/drivers/test_elasticsearch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from unittest import mock 17 | 18 | from osprofiler.drivers.elasticsearch_driver import ElasticsearchDriver 19 | from osprofiler.tests import test 20 | 21 | 22 | class ElasticsearchTestCase(test.TestCase): 23 | 24 | def setUp(self): 25 | super().setUp() 26 | self.elasticsearch = ElasticsearchDriver( 27 | "elasticsearch://localhost:9001/") 28 | self.elasticsearch.project = "project" 29 | self.elasticsearch.service = "service" 30 | 31 | def test_init_and_notify(self): 32 | self.elasticsearch.client = mock.MagicMock() 33 | self.elasticsearch.client.reset_mock() 34 | project = "project" 35 | service = "service" 36 | host = "host" 37 | 38 | info = { 39 | "a": 10, 40 | "project": project, 41 | "service": service, 42 | "host": host 43 | } 44 | self.elasticsearch.notify(info) 45 | 46 | self.elasticsearch.client\ 47 | .index.assert_called_once_with(index="osprofiler-notifications", 48 | doc_type="notification", 49 | body=info) 50 | 51 | def test_get_empty_report(self): 52 | self.elasticsearch.client = mock.MagicMock() 53 | self.elasticsearch.client.search = mock\ 54 | .MagicMock(return_value={"_scroll_id": "1", "hits": {"hits": []}}) 55 | self.elasticsearch.client.reset_mock() 56 | 57 | get_report = self.elasticsearch.get_report 58 | base_id = "abacaba" 59 | 60 | get_report(base_id) 61 | 62 | self.elasticsearch.client\ 63 | .search.assert_called_once_with(index="osprofiler-notifications", 64 | doc_type="notification", 65 | size=10000, 66 | scroll="2m", 67 | body={"query": { 68 | "match": {"base_id": base_id}} 69 | }) 70 | 71 | def test_get_non_empty_report(self): 72 | base_id = "1" 73 | elasticsearch_first_response = { 74 | "_scroll_id": "1", 75 | "hits": { 76 | "hits": [ 77 | { 78 | "_source": { 79 | "timestamp": "2016-08-10T16:58:03.064438", 80 | "base_id": base_id, 81 | "project": "project", 82 | "service": "service", 83 | "parent_id": "0", 84 | "name": "test", 85 | "info": { 86 | "host": "host" 87 | }, 88 | "trace_id": "1" 89 | } 90 | } 91 | ]}} 92 | elasticsearch_second_response = { 93 | "_scroll_id": base_id, 94 | "hits": {"hits": []}} 95 | self.elasticsearch.client = mock.MagicMock() 96 | self.elasticsearch.client.search = \ 97 | mock.MagicMock(return_value=elasticsearch_first_response) 98 | self.elasticsearch.client.scroll = \ 99 | mock.MagicMock(return_value=elasticsearch_second_response) 100 | 101 | self.elasticsearch.client.reset_mock() 102 | 103 | self.elasticsearch.get_report(base_id) 104 | 105 | self.elasticsearch.client\ 106 | .search.assert_called_once_with(index="osprofiler-notifications", 107 | doc_type="notification", 108 | size=10000, 109 | scroll="2m", 110 | body={"query": { 111 | "match": {"base_id": base_id}} 112 | }) 113 | 114 | self.elasticsearch.client\ 115 | .scroll.assert_called_once_with(scroll_id=base_id, scroll="2m") 116 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/drivers/test_messaging.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from unittest import mock 17 | 18 | from osprofiler.drivers import base 19 | from osprofiler.tests import test 20 | 21 | 22 | class MessagingTestCase(test.TestCase): 23 | 24 | @mock.patch("oslo_utils.importutils.try_import") 25 | def test_init_no_oslo_messaging(self, try_import_mock): 26 | try_import_mock.return_value = None 27 | 28 | self.assertRaises( 29 | ValueError, base.get_driver, 30 | "messaging://", project="project", service="service", 31 | host="host", context={}) 32 | 33 | @mock.patch("oslo_utils.importutils.try_import") 34 | def test_init_and_notify(self, try_import_mock): 35 | context = "context" 36 | transport = "transport" 37 | project = "project" 38 | service = "service" 39 | host = "host" 40 | 41 | # emulate dynamic load of oslo.messaging library 42 | oslo_messaging_mock = mock.Mock() 43 | try_import_mock.return_value = oslo_messaging_mock 44 | 45 | # mock oslo.messaging APIs 46 | notifier_mock = mock.Mock() 47 | oslo_messaging_mock.Notifier.return_value = notifier_mock 48 | oslo_messaging_mock.get_notification_transport.return_value = transport 49 | 50 | notify_func = base.get_driver( 51 | "messaging://", project=project, service=service, 52 | context=context, host=host).notify 53 | 54 | oslo_messaging_mock.Notifier.assert_called_once_with( 55 | transport, publisher_id=host, driver="messaging", 56 | topics=["profiler"], retry=0) 57 | 58 | info = { 59 | "a": 10, 60 | "project": project, 61 | "service": service, 62 | "host": host 63 | } 64 | notify_func(info) 65 | 66 | notifier_mock.info.assert_called_once_with( 67 | context, "profiler.service", info) 68 | 69 | notifier_mock.reset_mock() 70 | notify_func(info, context="my_context") 71 | notifier_mock.info.assert_called_once_with( 72 | "my_context", "profiler.service", info) 73 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/drivers/test_otlp.py: -------------------------------------------------------------------------------- 1 | # All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from unittest import mock 16 | 17 | from oslo_config import cfg 18 | 19 | from osprofiler.drivers import otlp 20 | from osprofiler import opts 21 | from osprofiler.tests import test 22 | 23 | 24 | class OTLPTestCase(test.TestCase): 25 | 26 | def setUp(self): 27 | super().setUp() 28 | 29 | opts.set_defaults(cfg.CONF) 30 | 31 | self.payload_start = { 32 | "name": "api-start", 33 | "base_id": "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee", 34 | "trace_id": "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1", 35 | "parent_id": "e2715537-3d1c-4f0c-b3af-87355dc5fc5b", 36 | "timestamp": "2018-05-03T04:31:51.781381", 37 | "info": { 38 | "host": "test" 39 | } 40 | } 41 | 42 | self.payload_stop = { 43 | "name": "api-stop", 44 | "base_id": "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee", 45 | "trace_id": "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1", 46 | "parent_id": "e2715537-3d1c-4f0c-b3af-87355dc5fc5b", 47 | "timestamp": "2018-05-03T04:31:51.781381", 48 | "info": { 49 | "host": "test", 50 | "function": { 51 | "result": 1 52 | } 53 | } 54 | } 55 | 56 | self.driver = otlp.OTLP( 57 | "otlp://127.0.0.1:6831", 58 | project="nova", service="api", 59 | conf=cfg.CONF) 60 | 61 | def test_notify_start(self): 62 | self.driver.notify(self.payload_start) 63 | self.assertEqual(1, len(self.driver.spans)) 64 | 65 | def test_notify_stop(self): 66 | mock_end = mock.MagicMock() 67 | self.driver.notify(self.payload_start) 68 | self.driver.spans[0].end = mock_end 69 | self.driver.notify(self.payload_stop) 70 | mock_end.assert_called_once() 71 | 72 | def test_service_name_default(self): 73 | self.assertEqual("pr1-svc1", self.driver._get_service_name( 74 | cfg.CONF, "pr1", "svc1")) 75 | 76 | def test_service_name_prefix(self): 77 | cfg.CONF.set_default( 78 | "service_name_prefix", "prx1", "profiler_otlp") 79 | self.assertEqual("prx1-pr1-svc1", self.driver._get_service_name( 80 | cfg.CONF, "pr1", "svc1")) 81 | 82 | def test_process_tags(self): 83 | # Need to be implemented. 84 | pass 85 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/test_initializer.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | from unittest import mock 14 | 15 | import testtools 16 | 17 | from osprofiler import initializer 18 | 19 | 20 | class InitializerTestCase(testtools.TestCase): 21 | 22 | @mock.patch("osprofiler.notifier.set") 23 | @mock.patch("osprofiler.notifier.create") 24 | @mock.patch("osprofiler.web.enable") 25 | def test_initializer(self, web_enable_mock, notifier_create_mock, 26 | notifier_set_mock): 27 | conf = mock.Mock() 28 | conf.profiler.connection_string = "driver://" 29 | conf.profiler.hmac_keys = "hmac_keys" 30 | context = {} 31 | project = "my-project" 32 | service = "my-service" 33 | host = "my-host" 34 | 35 | notifier_mock = mock.Mock() 36 | notifier_create_mock.return_value = notifier_mock 37 | 38 | initializer.init_from_conf(conf, context, project, service, host) 39 | 40 | notifier_create_mock.assert_called_once_with( 41 | "driver://", context=context, project=project, service=service, 42 | host=host, conf=conf) 43 | notifier_set_mock.assert_called_once_with(notifier_mock) 44 | web_enable_mock.assert_called_once_with("hmac_keys") 45 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/test_notifier.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from unittest import mock 17 | 18 | from osprofiler import notifier 19 | from osprofiler.tests import test 20 | 21 | 22 | class NotifierTestCase(test.TestCase): 23 | 24 | def tearDown(self): 25 | notifier.set(notifier._noop_notifier) # restore defaults 26 | notifier.clear_notifier_cache() 27 | super().tearDown() 28 | 29 | def test_set(self): 30 | 31 | def test(info): 32 | pass 33 | 34 | notifier.set(test) 35 | self.assertEqual(notifier.get(), test) 36 | 37 | def test_get_default_notifier(self): 38 | self.assertEqual(notifier.get(), notifier._noop_notifier) 39 | 40 | def test_notify(self): 41 | m = mock.MagicMock() 42 | notifier.set(m) 43 | notifier.notify(10) 44 | 45 | m.assert_called_once_with(10) 46 | 47 | @mock.patch("osprofiler.notifier.base.get_driver") 48 | def test_create(self, mock_factory): 49 | 50 | result = notifier.create("test", 10, b=20) 51 | mock_factory.assert_called_once_with("test", 10, b=20) 52 | self.assertEqual(mock_factory.return_value.notify, result) 53 | 54 | @mock.patch("osprofiler.notifier.base.get_driver") 55 | def test_create_driver_init_failure(self, mock_get_driver): 56 | mock_get_driver.side_effect = Exception() 57 | 58 | result = notifier.create("test", 10, b=20) 59 | mock_get_driver.assert_called_once_with("test", 10, b=20) 60 | self.assertEqual(notifier._noop_notifier, result) 61 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/test_opts.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | from unittest import mock 17 | 18 | from oslo_config import fixture 19 | 20 | from osprofiler import opts 21 | from osprofiler.tests import test 22 | 23 | 24 | class ConfigTestCase(test.TestCase): 25 | def setUp(self): 26 | super().setUp() 27 | self.conf_fixture = self.useFixture(fixture.Config()) 28 | 29 | def test_options_defaults(self): 30 | opts.set_defaults(self.conf_fixture.conf) 31 | self.assertFalse(self.conf_fixture.conf.profiler.enabled) 32 | self.assertFalse(self.conf_fixture.conf.profiler.trace_sqlalchemy) 33 | self.assertEqual("SECRET_KEY", 34 | self.conf_fixture.conf.profiler.hmac_keys) 35 | self.assertFalse(opts.is_trace_enabled(self.conf_fixture.conf)) 36 | self.assertFalse(opts.is_db_trace_enabled(self.conf_fixture.conf)) 37 | 38 | def test_options_defaults_override(self): 39 | opts.set_defaults(self.conf_fixture.conf, enabled=True, 40 | trace_sqlalchemy=True, 41 | hmac_keys="MY_KEY") 42 | self.assertTrue(self.conf_fixture.conf.profiler.enabled) 43 | self.assertTrue(self.conf_fixture.conf.profiler.trace_sqlalchemy) 44 | self.assertEqual("MY_KEY", 45 | self.conf_fixture.conf.profiler.hmac_keys) 46 | self.assertTrue(opts.is_trace_enabled(self.conf_fixture.conf)) 47 | self.assertTrue(opts.is_db_trace_enabled(self.conf_fixture.conf)) 48 | 49 | @mock.patch("osprofiler.web.enable") 50 | @mock.patch("osprofiler.web.disable") 51 | def test_web_trace_disabled(self, mock_disable, mock_enable): 52 | opts.set_defaults(self.conf_fixture.conf, hmac_keys="MY_KEY") 53 | opts.enable_web_trace(self.conf_fixture.conf) 54 | opts.disable_web_trace(self.conf_fixture.conf) 55 | self.assertEqual(0, mock_enable.call_count) 56 | self.assertEqual(0, mock_disable.call_count) 57 | 58 | @mock.patch("osprofiler.web.enable") 59 | @mock.patch("osprofiler.web.disable") 60 | def test_web_trace_enabled(self, mock_disable, mock_enable): 61 | opts.set_defaults(self.conf_fixture.conf, enabled=True, 62 | hmac_keys="MY_KEY") 63 | opts.enable_web_trace(self.conf_fixture.conf) 64 | opts.disable_web_trace(self.conf_fixture.conf) 65 | mock_enable.assert_called_once_with("MY_KEY") 66 | mock_disable.assert_called_once_with() 67 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/test_sqlalchemy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import contextlib 17 | from unittest import mock 18 | 19 | 20 | from osprofiler import sqlalchemy 21 | from osprofiler.tests import test 22 | 23 | 24 | class SqlalchemyTracingTestCase(test.TestCase): 25 | 26 | @mock.patch("osprofiler.sqlalchemy.profiler") 27 | def test_before_execute(self, mock_profiler): 28 | handler = sqlalchemy._before_cursor_execute("sql") 29 | 30 | handler(mock.MagicMock(), 1, 2, 3, 4, 5) 31 | expected_info = {"db": {"statement": 2, "params": 3}} 32 | mock_profiler.start.assert_called_once_with("sql", info=expected_info) 33 | 34 | @mock.patch("osprofiler.sqlalchemy.profiler") 35 | def test_after_execute(self, mock_profiler): 36 | handler = sqlalchemy._after_cursor_execute() 37 | handler(mock.MagicMock(), 1, 2, 3, 4, 5) 38 | mock_profiler.stop.assert_called_once_with() 39 | 40 | @mock.patch("osprofiler.sqlalchemy.profiler") 41 | def test_after_execute_with_sql_result(self, mock_profiler): 42 | handler = sqlalchemy._after_cursor_execute(hide_result=False) 43 | cursor = mock.MagicMock() 44 | cursor._rows = (1,) 45 | handler(1, cursor, 2, 3, 4, 5) 46 | info = { 47 | "db": { 48 | "result": str(cursor._rows) 49 | } 50 | } 51 | mock_profiler.stop.assert_called_once_with(info=info) 52 | 53 | @mock.patch("osprofiler.sqlalchemy.profiler") 54 | def test_handle_error(self, mock_profiler): 55 | original_exception = Exception("error") 56 | chained_exception = Exception("error and the reason") 57 | 58 | sqlalchemy_exception_ctx = mock.MagicMock() 59 | sqlalchemy_exception_ctx.original_exception = original_exception 60 | sqlalchemy_exception_ctx.chained_exception = chained_exception 61 | 62 | sqlalchemy.handle_error(sqlalchemy_exception_ctx) 63 | expected_info = { 64 | "etype": "Exception", 65 | "message": "error", 66 | "db": { 67 | "original_exception": str(original_exception), 68 | "chained_exception": str(chained_exception), 69 | } 70 | } 71 | mock_profiler.stop.assert_called_once_with(info=expected_info) 72 | 73 | @mock.patch("osprofiler.sqlalchemy.handle_error") 74 | @mock.patch("osprofiler.sqlalchemy._before_cursor_execute") 75 | @mock.patch("osprofiler.sqlalchemy._after_cursor_execute") 76 | def test_add_tracing(self, mock_after_exc, mock_before_exc, 77 | mock_handle_error): 78 | sa = mock.MagicMock() 79 | engine = mock.MagicMock() 80 | 81 | mock_before_exc.return_value = "before" 82 | mock_after_exc.return_value = "after" 83 | 84 | sqlalchemy.add_tracing(sa, engine, "sql") 85 | 86 | mock_before_exc.assert_called_once_with("sql") 87 | # Default set hide_result=True 88 | mock_after_exc.assert_called_once_with(hide_result=True) 89 | expected_calls = [ 90 | mock.call(engine, "before_cursor_execute", "before"), 91 | mock.call(engine, "after_cursor_execute", "after"), 92 | mock.call(engine, "handle_error", mock_handle_error), 93 | ] 94 | self.assertEqual(sa.event.listen.call_args_list, expected_calls) 95 | 96 | @mock.patch("osprofiler.sqlalchemy.handle_error") 97 | @mock.patch("osprofiler.sqlalchemy._before_cursor_execute") 98 | @mock.patch("osprofiler.sqlalchemy._after_cursor_execute") 99 | def test_wrap_session(self, mock_after_exc, mock_before_exc, 100 | mock_handle_error): 101 | sa = mock.MagicMock() 102 | 103 | @contextlib.contextmanager 104 | def _session(): 105 | session = mock.MagicMock() 106 | # current engine object stored within the session 107 | session.bind = mock.MagicMock() 108 | session.bind.traced = None 109 | yield session 110 | 111 | mock_before_exc.return_value = "before" 112 | mock_after_exc.return_value = "after" 113 | 114 | session = sqlalchemy.wrap_session(sa, _session()) 115 | 116 | with session as sess: 117 | pass 118 | 119 | mock_before_exc.assert_called_once_with("db") 120 | # Default set hide_result=True 121 | mock_after_exc.assert_called_once_with(hide_result=True) 122 | expected_calls = [ 123 | mock.call(sess.bind, "before_cursor_execute", "before"), 124 | mock.call(sess.bind, "after_cursor_execute", "after"), 125 | mock.call(sess.bind, "handle_error", mock_handle_error), 126 | ] 127 | 128 | self.assertEqual(sa.event.listen.call_args_list, expected_calls) 129 | 130 | @mock.patch("osprofiler.sqlalchemy.handle_error") 131 | @mock.patch("osprofiler.sqlalchemy._before_cursor_execute") 132 | @mock.patch("osprofiler.sqlalchemy._after_cursor_execute") 133 | @mock.patch("osprofiler.profiler") 134 | def test_with_sql_result(self, mock_profiler, mock_after_exc, 135 | mock_before_exc, mock_handle_error): 136 | sa = mock.MagicMock() 137 | engine = mock.MagicMock() 138 | 139 | mock_before_exc.return_value = "before" 140 | mock_after_exc.return_value = "after" 141 | 142 | sqlalchemy.add_tracing(sa, engine, "sql", hide_result=False) 143 | 144 | mock_before_exc.assert_called_once_with("sql") 145 | # Default set hide_result=True 146 | mock_after_exc.assert_called_once_with(hide_result=False) 147 | expected_calls = [ 148 | mock.call(engine, "before_cursor_execute", "before"), 149 | mock.call(engine, "after_cursor_execute", "after"), 150 | mock.call(engine, "handle_error", mock_handle_error), 151 | ] 152 | self.assertEqual(sa.event.listen.call_args_list, expected_calls) 153 | 154 | @mock.patch("osprofiler.sqlalchemy._before_cursor_execute") 155 | @mock.patch("osprofiler.sqlalchemy._after_cursor_execute") 156 | def test_disable_and_enable(self, mock_after_exc, mock_before_exc): 157 | sqlalchemy.disable() 158 | 159 | sa = mock.MagicMock() 160 | engine = mock.MagicMock() 161 | sqlalchemy.add_tracing(sa, engine, "sql") 162 | self.assertFalse(mock_after_exc.called) 163 | self.assertFalse(mock_before_exc.called) 164 | 165 | sqlalchemy.enable() 166 | sqlalchemy.add_tracing(sa, engine, "sql") 167 | self.assertTrue(mock_after_exc.called) 168 | self.assertTrue(mock_before_exc.called) 169 | -------------------------------------------------------------------------------- /osprofiler/tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import base64 17 | import hashlib 18 | import hmac 19 | from unittest import mock 20 | import uuid 21 | 22 | 23 | from osprofiler import _utils as utils 24 | from osprofiler.tests import test 25 | 26 | 27 | class UtilsTestCase(test.TestCase): 28 | 29 | def test_split(self): 30 | self.assertEqual([1, 2], utils.split([1, 2])) 31 | self.assertEqual(["A", "B"], utils.split("A, B")) 32 | self.assertEqual(["A", " B"], utils.split("A, B", strip=False)) 33 | 34 | def test_split_wrong_type(self): 35 | self.assertRaises(TypeError, utils.split, 1) 36 | 37 | def test_binary_encode_and_decode(self): 38 | self.assertEqual("text", 39 | utils.binary_decode(utils.binary_encode("text"))) 40 | 41 | def test_binary_encode_invalid_type(self): 42 | self.assertRaises(TypeError, utils.binary_encode, 1234) 43 | 44 | def test_binary_encode_binary_type(self): 45 | binary = utils.binary_encode("text") 46 | self.assertEqual(binary, utils.binary_encode(binary)) 47 | 48 | def test_binary_decode_invalid_type(self): 49 | self.assertRaises(TypeError, utils.binary_decode, 1234) 50 | 51 | def test_binary_decode_text_type(self): 52 | self.assertEqual("text", utils.binary_decode("text")) 53 | 54 | def test_generate_hmac(self): 55 | hmac_key = "secrete" 56 | data = "my data" 57 | 58 | h = hmac.new(utils.binary_encode(hmac_key), digestmod=hashlib.sha1) 59 | h.update(utils.binary_encode(data)) 60 | 61 | self.assertEqual(h.hexdigest(), utils.generate_hmac(data, hmac_key)) 62 | 63 | def test_signed_pack_unpack(self): 64 | hmac = "secret" 65 | data = {"some": "data"} 66 | 67 | packed_data, hmac_data = utils.signed_pack(data, hmac) 68 | 69 | process_data = utils.signed_unpack(packed_data, hmac_data, [hmac]) 70 | self.assertIn("hmac_key", process_data) 71 | process_data.pop("hmac_key") 72 | self.assertEqual(data, process_data) 73 | 74 | def test_signed_pack_unpack_many_keys(self): 75 | keys = ["secret", "secret2", "secret3"] 76 | data = {"some": "data"} 77 | packed_data, hmac_data = utils.signed_pack(data, keys[-1]) 78 | 79 | process_data = utils.signed_unpack(packed_data, hmac_data, keys) 80 | self.assertEqual(keys[-1], process_data["hmac_key"]) 81 | 82 | def test_signed_pack_unpack_many_wrong_keys(self): 83 | keys = ["secret", "secret2", "secret3"] 84 | data = {"some": "data"} 85 | packed_data, hmac_data = utils.signed_pack(data, "password") 86 | 87 | process_data = utils.signed_unpack(packed_data, hmac_data, keys) 88 | self.assertIsNone(process_data) 89 | 90 | def test_signed_unpack_wrong_key(self): 91 | data = {"some": "data"} 92 | packed_data, hmac_data = utils.signed_pack(data, "secret") 93 | 94 | self.assertIsNone(utils.signed_unpack(packed_data, hmac_data, "wrong")) 95 | 96 | def test_signed_unpack_no_key_or_hmac_data(self): 97 | data = {"some": "data"} 98 | packed_data, hmac_data = utils.signed_pack(data, "secret") 99 | self.assertIsNone(utils.signed_unpack(packed_data, hmac_data, None)) 100 | self.assertIsNone(utils.signed_unpack(packed_data, None, "secret")) 101 | self.assertIsNone(utils.signed_unpack(packed_data, " ", "secret")) 102 | 103 | @mock.patch("osprofiler._utils.generate_hmac") 104 | def test_singed_unpack_generate_hmac_failed(self, mock_generate_hmac): 105 | mock_generate_hmac.side_effect = Exception 106 | self.assertIsNone(utils.signed_unpack("data", "hmac_data", "hmac_key")) 107 | 108 | def test_signed_unpack_invalid_json(self): 109 | hmac = "secret" 110 | data = base64.urlsafe_b64encode(utils.binary_encode("not_a_json")) 111 | hmac_data = utils.generate_hmac(data, hmac) 112 | 113 | self.assertIsNone(utils.signed_unpack(data, hmac_data, hmac)) 114 | 115 | def test_shorten_id_with_valid_uuid(self): 116 | valid_id = "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee" 117 | 118 | uuid_obj = uuid.UUID(valid_id) 119 | 120 | with mock.patch("uuid.UUID") as mock_uuid: 121 | mock_uuid.return_value = uuid_obj 122 | 123 | result = utils.shorten_id(valid_id) 124 | expected = 9584796812364680686 125 | 126 | self.assertEqual(expected, result) 127 | 128 | @mock.patch("oslo_utils.uuidutils.generate_uuid") 129 | def test_shorten_id_with_invalid_uuid(self, mock_gen_uuid): 130 | invalid_id = "invalid" 131 | mock_gen_uuid.return_value = "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1" 132 | 133 | result = utils.shorten_id(invalid_id) 134 | expected = 10088334584203457265 135 | 136 | self.assertEqual(expected, result) 137 | 138 | def test_itersubclasses(self): 139 | 140 | class A: 141 | pass 142 | 143 | class B(A): 144 | pass 145 | 146 | class C(A): 147 | pass 148 | 149 | class D(C): 150 | pass 151 | 152 | self.assertEqual([B, C, D], list(utils.itersubclasses(A))) 153 | 154 | class E(type): 155 | pass 156 | 157 | self.assertEqual([], list(utils.itersubclasses(E))) 158 | -------------------------------------------------------------------------------- /osprofiler/web.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Mirantis Inc. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import webob.dec 17 | 18 | from osprofiler import _utils as utils 19 | from osprofiler import profiler 20 | 21 | 22 | # Trace keys that are required or optional, any other 23 | # keys that are present will cause the trace to be rejected... 24 | _REQUIRED_KEYS = ("base_id", "hmac_key") 25 | _OPTIONAL_KEYS = ("parent_id",) 26 | 27 | #: Http header that will contain the needed traces data. 28 | X_TRACE_INFO = "X-Trace-Info" 29 | 30 | #: Http header that will contain the traces data hmac (that will be validated). 31 | X_TRACE_HMAC = "X-Trace-HMAC" 32 | 33 | 34 | def get_trace_id_headers(): 35 | """Adds the trace id headers (and any hmac) into provided dictionary.""" 36 | p = profiler.get() 37 | if p and p.hmac_key: 38 | data = {"base_id": p.get_base_id(), "parent_id": p.get_id()} 39 | pack = utils.signed_pack(data, p.hmac_key) 40 | return { 41 | X_TRACE_INFO: pack[0], 42 | X_TRACE_HMAC: pack[1] 43 | } 44 | return {} 45 | 46 | 47 | _ENABLED = None 48 | _HMAC_KEYS = None 49 | 50 | 51 | def disable(): 52 | """Disable middleware. 53 | 54 | This is the alternative way to disable middleware. It will be used to be 55 | able to disable middleware via oslo.config. 56 | """ 57 | global _ENABLED 58 | _ENABLED = False 59 | 60 | 61 | def enable(hmac_keys=None): 62 | """Enable middleware.""" 63 | global _ENABLED, _HMAC_KEYS 64 | _ENABLED = True 65 | _HMAC_KEYS = utils.split(hmac_keys or "") 66 | 67 | 68 | class WsgiMiddleware: 69 | """WSGI Middleware that enables tracing for an application.""" 70 | 71 | def __init__(self, application, hmac_keys=None, enabled=False, **kwargs): 72 | """Initialize middleware with api-paste.ini arguments. 73 | 74 | :application: wsgi app 75 | :hmac_keys: Only trace header that was signed with one of these 76 | hmac keys will be processed. This limitation is 77 | essential, because it allows to profile OpenStack 78 | by only those who knows this key which helps 79 | avoid DDOS. 80 | :enabled: This middleware can be turned off fully if enabled is False. 81 | :kwargs: Other keyword arguments. 82 | NOTE(tovin07): Currently, this `kwargs` is not used at all. 83 | It's here to avoid some extra keyword arguments in local_conf 84 | that cause `__init__() got an unexpected keyword argument`. 85 | """ 86 | self.application = application 87 | self.name = "wsgi" 88 | self.enabled = enabled 89 | self.hmac_keys = utils.split(hmac_keys or "") 90 | 91 | @classmethod 92 | def factory(cls, global_conf, **local_conf): 93 | def filter_(app): 94 | return cls(app, **local_conf) 95 | return filter_ 96 | 97 | def _trace_is_valid(self, trace_info): 98 | if not isinstance(trace_info, dict): 99 | return False 100 | trace_keys = set(trace_info.keys()) 101 | if not all(k in trace_keys for k in _REQUIRED_KEYS): 102 | return False 103 | if trace_keys.difference(_REQUIRED_KEYS + _OPTIONAL_KEYS): 104 | return False 105 | return True 106 | 107 | @webob.dec.wsgify 108 | def __call__(self, request): 109 | if (_ENABLED is not None and not _ENABLED 110 | or _ENABLED is None and not self.enabled): 111 | return request.get_response(self.application) 112 | 113 | trace_info = utils.signed_unpack(request.headers.get(X_TRACE_INFO), 114 | request.headers.get(X_TRACE_HMAC), 115 | _HMAC_KEYS or self.hmac_keys) 116 | 117 | if not self._trace_is_valid(trace_info): 118 | return request.get_response(self.application) 119 | 120 | profiler.init(**trace_info) 121 | info = { 122 | "request": { 123 | "path": request.path, 124 | "query": request.query_string, 125 | "method": request.method, 126 | "scheme": request.scheme 127 | } 128 | } 129 | try: 130 | with profiler.Trace(self.name, info=info): 131 | return request.get_response(self.application) 132 | finally: 133 | profiler.clean() 134 | -------------------------------------------------------------------------------- /playbooks/osprofiler-post.yaml: -------------------------------------------------------------------------------- 1 | - hosts: controller 2 | vars: 3 | osprofiler_traces_dir: '/opt/stack/osprofiler-traces' 4 | tasks: 5 | - name: Create directory for traces 6 | become: True 7 | become_user: stack 8 | file: 9 | path: '{{ osprofiler_traces_dir }}' 10 | state: directory 11 | owner: stack 12 | group: stack 13 | 14 | - name: Read connection string from a file 15 | command: "cat /opt/stack/.osprofiler_connection_string" 16 | register: osprofiler_connection_string 17 | 18 | - debug: 19 | msg: "OSProfiler connection string is: {{ osprofiler_connection_string.stdout }}" 20 | 21 | - name: Get list of traces 22 | command: "osprofiler trace list --connection-string {{ osprofiler_connection_string.stdout }}" 23 | become: True 24 | become_user: stack 25 | register: osprofiler_trace_list 26 | 27 | - debug: 28 | msg: "{{ osprofiler_trace_list }}" 29 | 30 | - name: Save traces to files 31 | shell: | 32 | osprofiler trace list --connection-string {{ osprofiler_connection_string.stdout }} > {{ osprofiler_traces_dir }}/trace_list.txt 33 | cat {{ osprofiler_traces_dir }}/trace_list.txt | tail -n +4 | head -n -1 | awk '{print $2}' > {{ osprofiler_traces_dir }}/trace_ids.txt 34 | 35 | while read p; do 36 | osprofiler trace show --connection-string {{ osprofiler_connection_string.stdout }} --html $p > {{ osprofiler_traces_dir }}/trace-$p.html 37 | done < {{ osprofiler_traces_dir }}/trace_ids.txt 38 | become: True 39 | become_user: stack 40 | 41 | - name: Gzip trace files 42 | become: yes 43 | become_user: stack 44 | shell: "gzip * -9 -q | true" 45 | args: 46 | chdir: '{{ osprofiler_traces_dir }}' 47 | 48 | - name: Sync trace files to Zuul 49 | become: yes 50 | synchronize: 51 | src: "{{ osprofiler_traces_dir }}" 52 | dest: "{{ zuul.executor.log_root }}" 53 | mode: pull 54 | copy_links: true 55 | verify_host: true 56 | rsync_opts: 57 | - "--include=/**" 58 | - "--include=*/" 59 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pbr>=6.1.1"] 3 | build-backend = "pbr.build" 4 | -------------------------------------------------------------------------------- /releasenotes/notes/add-reno-996dd44974d53238.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | other: 3 | - Introduce reno for deployer release notes. 4 | -------------------------------------------------------------------------------- /releasenotes/notes/add-requests-profiling-761e09f243d36966.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | New profiler for python requests. Currently only OTLP driver is 5 | supporting it, see `profiler/trace_requests`'s option. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/drop-jaeger-container-when-unstacking-e8fcdc036f80158a.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | issues: 3 | - | 4 | Using devstack, when unstacking, the docker container running 5 | Jaeger tracing will be deleted that to correctly clean processes 6 | started by devstack.. This also avoid `./stack.sh` to fail when 7 | recreating the environnement. 8 | -------------------------------------------------------------------------------- /releasenotes/notes/drop-python-2-7-73d3113c69d724d6.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | Python 2.7 support has been dropped. The minimum version of Python now 5 | supported by osprofiler is Python 3.6. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/jaeger-add-process-tags-79d5f5d7a0b049ef.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Add ability to set tracer process tags to Jaeger via a 5 | configuration option introduced, `profiler_jaeger/process_tags`. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/jaeger-service-name-prefix-72878a930f700878.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Introduces service name prefix for Jaeger driver. Please consider 5 | using option `profiler_jaeger/service_name_prefix` to set it. 6 | -------------------------------------------------------------------------------- /releasenotes/notes/otlp-driver-cb932038ad580ac2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | An OTLP (OpenTelemetry) exporter is now supported. The current 5 | support is experimental but the aim is to deprecate and remove 6 | legacy Jaeger driver which is using the already deprecated python 7 | library jaeger client. Operators who want to use it should enable 8 | `otlp`. OTLP is comptatible with Jaeger backend. 9 | -------------------------------------------------------------------------------- /releasenotes/notes/redis-improvement-d4c91683fc89f570.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | features: 3 | - | 4 | Redis storage schema is optimized for higher performance. 5 | Previously Redis driver stored each tracing event under its own key, 6 | as result both list and get operations required full scan of the database. 7 | With the optimized schema traces are stored as Redis lists under a key 8 | equal to trace id. So list operation iterates only over unique 9 | trace ids and get operation retrieves content of a specified list. 10 | Note that list operation still needs to retrieve at least 1 event 11 | from the trace to get a timestamp. 12 | upgrade: 13 | - | 14 | The optimized Redis driver is backward compatible: while new events are stored 15 | using new schema the driver can retrieve existing events using both old and new 16 | schemas. 17 | -------------------------------------------------------------------------------- /releasenotes/notes/remove-py38-e2c2723282ebbf9f.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | Support for Python 3.8 has been removed. Now the minimum python version 5 | supported is 3.9 . 6 | -------------------------------------------------------------------------------- /releasenotes/notes/remove-strict-redis-9eb43d30c9c1fc43.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | The minimum redis-py version required is now >= 3.0.0 5 | -------------------------------------------------------------------------------- /releasenotes/notes/retire-jaeger-driver-d8add44c5522ad7a.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | upgrade: 3 | - | 4 | The Jager driver is no longer supported and now using it raises 5 | CommandError. This is because the jaeger-client-python library, which is 6 | its core dependency, was already retired. 7 | -------------------------------------------------------------------------------- /releasenotes/source/_static/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/releasenotes/source/_static/.placeholder -------------------------------------------------------------------------------- /releasenotes/source/_templates/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/osprofiler/e03de733da7d4d9af5d538081277387d31e93d28/releasenotes/source/_templates/.placeholder -------------------------------------------------------------------------------- /releasenotes/source/index.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | osprofiler Release Notes 3 | ========================== 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | unreleased 9 | victoria 10 | ussuri 11 | train 12 | stein 13 | rocky 14 | queens 15 | pike 16 | ocata 17 | -------------------------------------------------------------------------------- /releasenotes/source/ocata.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Ocata Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: origin/stable/ocata 7 | -------------------------------------------------------------------------------- /releasenotes/source/pike.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Pike Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/pike 7 | -------------------------------------------------------------------------------- /releasenotes/source/queens.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Queens Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/queens 7 | -------------------------------------------------------------------------------- /releasenotes/source/rocky.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Rocky Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/rocky 7 | -------------------------------------------------------------------------------- /releasenotes/source/stein.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | Stein Series Release Notes 3 | =================================== 4 | 5 | .. release-notes:: 6 | :branch: stable/stein 7 | -------------------------------------------------------------------------------- /releasenotes/source/train.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | Train Series Release Notes 3 | ========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/train 7 | -------------------------------------------------------------------------------- /releasenotes/source/unreleased.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | Current Series Release Notes 3 | ============================== 4 | 5 | .. release-notes:: 6 | -------------------------------------------------------------------------------- /releasenotes/source/ussuri.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Ussuri Series Release Notes 3 | =========================== 4 | 5 | .. release-notes:: 6 | :branch: stable/ussuri 7 | -------------------------------------------------------------------------------- /releasenotes/source/victoria.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | Victoria Series Release Notes 3 | ============================= 4 | 5 | .. release-notes:: 6 | :branch: unmaintained/victoria 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | netaddr>=0.7.18 # BSD 2 | oslo.concurrency>=3.26.0 # Apache-2.0 3 | oslo.config>=5.2.0 # Apache-2.0 4 | oslo.serialization>=2.18.0 # Apache-2.0 5 | oslo.utils>=3.33.0 # Apache-2.0 6 | PrettyTable>=0.7.2 # BSD 7 | requests>=2.14.2 # Apache-2.0 8 | WebOb>=1.7.1 # MIT 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = osprofiler 3 | summary = OpenStack Profiler Library 4 | description_file = 5 | README.rst 6 | author = OpenStack 7 | author_email = openstack-discuss@lists.openstack.org 8 | home_page = https://docs.openstack.org/osprofiler/latest/ 9 | python_requires = >=3.9 10 | classifier = 11 | Environment :: OpenStack 12 | Intended Audience :: Developers 13 | Intended Audience :: Information Technology 14 | License :: OSI Approved :: Apache Software License 15 | Operating System :: POSIX :: Linux 16 | Programming Language :: Python 17 | Programming Language :: Python :: 3 18 | Programming Language :: Python :: 3.9 19 | Programming Language :: Python :: 3.10 20 | Programming Language :: Python :: 3.11 21 | Programming Language :: Python :: 3.12 22 | Programming Language :: Python :: 3 :: Only 23 | Programming Language :: Python :: Implementation :: CPython 24 | 25 | [files] 26 | packages = 27 | osprofiler 28 | 29 | [extras] 30 | elasticsearch = 31 | elasticsearch>=2.0.0 # Apache-2.0 32 | messaging = 33 | oslo.messaging>=14.1.0 # Apache-2.0 34 | mongo = 35 | pymongo!=3.1,>=3.0.2 # Apache-2.0 36 | otlp = 37 | opentelemetry-exporter-otlp>=1.16.0 #Apache-2.0 38 | opentelemetry-sdk>=1.16.0 # Apache-2.0 39 | redis = 40 | redis>=2.10.0 # MIT 41 | sqlalchemy = 42 | SQLAlchemy>=1.4.0 # MIT 43 | 44 | [entry_points] 45 | oslo.config.opts = 46 | osprofiler = osprofiler.opts:list_opts 47 | console_scripts = 48 | osprofiler = osprofiler.cmd.shell:main 49 | paste.filter_factory = 50 | osprofiler = osprofiler.web:WsgiMiddleware.factory 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | # implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT 15 | import setuptools 16 | 17 | setuptools.setup( 18 | setup_requires=['pbr>=2.0'], 19 | pbr=True) 20 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | coverage>=4.0 # Apache-2.0 2 | ddt>=1.0.1 # MIT 3 | stestr>=2.0.0 # Apache-2.0 4 | testtools>=2.2.0 # MIT 5 | docutils>=0.14 # OSI-Approved Open Source, Public Domain 6 | 7 | pymongo!=3.1,>=3.0.2 # Apache-2.0 8 | 9 | # Elasticsearch python client 10 | elasticsearch>=2.0.0 # Apache-2.0 11 | 12 | # Redis python client 13 | redis>=2.10.0 # MIT 14 | 15 | # For OTLP 16 | opentelemetry-exporter-otlp>=1.16.0 # Apache-2.0 17 | opentelemetry-sdk>=1.16.0 # Apache-2.0 18 | -------------------------------------------------------------------------------- /tools/lint.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2013 Intel Corporation. 2 | # All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | # not use this file except in compliance with the License. You may obtain 6 | # a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations 14 | # under the License. 15 | 16 | import sys 17 | 18 | from pylint import lint 19 | 20 | 21 | ENABLED_PYLINT_MSGS = ['W0611'] 22 | 23 | 24 | def main(dirpath): 25 | enable_opt = '--enable=%s' % ','.join(ENABLED_PYLINT_MSGS) 26 | lint.Run(['--reports=n', '--disable=all', enable_opt, dirpath]) 27 | 28 | 29 | if __name__ == '__main__': 30 | main(sys.argv[1]) 31 | -------------------------------------------------------------------------------- /tools/patch_tox_venv.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | import os 16 | import sys 17 | 18 | import install_venv_common as install_venv # noqa 19 | 20 | 21 | def first_file(file_list): 22 | for candidate in file_list: 23 | if os.path.exists(candidate): 24 | return candidate 25 | 26 | 27 | def main(argv): 28 | root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 29 | 30 | venv = os.environ['VIRTUAL_ENV'] 31 | 32 | pip_requires = first_file([ 33 | os.path.join(root, 'requirements.txt'), 34 | os.path.join(root, 'tools', 'pip-requires'), 35 | ]) 36 | test_requires = first_file([ 37 | os.path.join(root, 'test-requirements.txt'), 38 | os.path.join(root, 'tools', 'test-requires'), 39 | ]) 40 | py_version = "python{}.{}".format(sys.version_info[0], sys.version_info[1]) 41 | project = 'oslo' 42 | install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, 43 | py_version, project) 44 | # NOTE(dprince): For Tox we only run post_process, which patches files, etc 45 | install.post_process() 46 | 47 | 48 | if __name__ == '__main__': 49 | main(sys.argv) 50 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 3.18.0 3 | envlist = py3,pep8 4 | 5 | [testenv] 6 | setenv = 7 | LANG=en_US.UTF-8 8 | LANGUAGE=en_US:en 9 | LC_ALL=C 10 | deps = 11 | -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} 12 | -r{toxinidir}/requirements.txt 13 | -r{toxinidir}/test-requirements.txt 14 | usedevelop = True 15 | commands = stestr run --slowest {posargs} 16 | distribute = false 17 | 18 | [testenv:functional{,-py38,-py39}] 19 | setenv = 20 | {[testenv]setenv} 21 | OS_TEST_PATH=./osprofiler/tests/functional 22 | deps = 23 | {[testenv]deps} 24 | oslo.messaging 25 | 26 | [testenv:pep8] 27 | deps = 28 | pre-commit 29 | commands = 30 | pre-commit run -a 31 | 32 | [testenv:venv] 33 | commands = {posargs} 34 | 35 | [testenv:cover] 36 | setenv = 37 | PYTHON=coverage run --source osprofiler --parallel-mode 38 | commands = 39 | stestr run {posargs} 40 | coverage combine 41 | coverage html -d cover 42 | coverage xml -o cover/coverage.xml 43 | 44 | [testenv:docs] 45 | deps = 46 | -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} 47 | -r{toxinidir}/requirements.txt 48 | -r{toxinidir}/doc/requirements.txt 49 | allowlist_externals = rm 50 | commands = 51 | rm -rf doc/build api-guide/build api-ref/build doc/source/contributor/modules 52 | sphinx-build -W --keep-going -b html -d doc/build/doctrees doc/source doc/build/html 53 | usedevelop = false 54 | 55 | [flake8] 56 | show-source = true 57 | builtins = _ 58 | # E741 ambiguous variable name 'l' 59 | # W503 line break before binary operator 60 | ignore = E741,W503 61 | exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,setup.py,build,releasenotes 62 | import-order-style = pep8 63 | application-import-names = osprofiler 64 | 65 | [flake8:local-plugins] 66 | extension = 67 | N301 = checks:check_assert_methods_from_mock 68 | N320 = checks:assert_true_instance 69 | N321 = checks:assert_equal_type 70 | N322 = checks:assert_equal_none 71 | N323 = checks:assert_true_or_false_with_in 72 | N324 = checks:assert_equal_in 73 | N351 = checks:check_no_constructor_data_struct 74 | N352 = checks:check_dict_formatting_in_string 75 | N353 = checks:check_using_unicode 76 | N354 = checks:check_raises 77 | paths = ./osprofiler/hacking 78 | 79 | [testenv:releasenotes] 80 | deps = 81 | -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} 82 | -r{toxinidir}/doc/requirements.txt 83 | allowlist_externals = rm 84 | commands = 85 | rm -rf releasenotes/build 86 | sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html 87 | 88 | [testenv:lower-constraints] 89 | deps = 90 | -c{toxinidir}/lower-constraints.txt 91 | -r{toxinidir}/test-requirements.txt 92 | -r{toxinidir}/requirements.txt 93 | --------------------------------------------------------------------------------