├── requirements.devel.txt ├── docs ├── authors.rst ├── readme.rst ├── contributing.rst ├── history.rst ├── config.rst ├── index.rst ├── statsd.rst ├── prometheus.rst ├── flask.rst ├── talisker_config.py ├── celery.rst ├── endpoints.rst ├── development.rst ├── gunicorn.rst ├── timeouts.rst ├── postgresql.rst ├── django.rst ├── context.rst ├── sentry.rst ├── overview.rst ├── requests.rst ├── testing.rst ├── Makefile └── make.bat ├── requirements.lint.txt ├── requirements.docs.txt ├── dependencies.txt ├── TODO.rst ├── talisker ├── logstash │ ├── talisker.pattern │ └── talisker.filter ├── __main__.py ├── django.py ├── statsd.py ├── metrics.py ├── flask.py └── postgresql.py ├── .bumpversion.cfg ├── MANIFEST.in ├── AUTHORS.rst ├── .gitignore ├── tests ├── access.log ├── test_truncate.log ├── test.log ├── django_app │ ├── __init__.py │ ├── django_app │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ ├── __init__.py │ │ │ │ └── test_command.py │ │ ├── migrations │ │ │ ├── __init__.py │ │ │ └── 0001_create_procedure.py │ │ ├── __init__.py │ │ ├── wsgi.py │ │ ├── celery.py │ │ ├── views.py │ │ ├── urls.py │ │ └── settings.py │ └── manage.py ├── __init__.py ├── py36_async_tls.py ├── udpecho.py ├── logstash-talisker.yaml ├── statsd.sh ├── test_django.py ├── wsgi_app.py ├── flask_app.py ├── celery_app.py ├── test_metrics.py ├── test_init.py ├── test_statsd.py ├── test_render.py ├── test_postgresql.py ├── conftest.py ├── test_celery.py ├── test_util.py ├── test_flask.py └── test_testing.py ├── HACKING ├── LICENSE ├── .readthedocs.yaml ├── .github └── workflows │ └── tox.yml ├── .pyup.yml ├── requirements.tests.txt ├── tox.ini ├── README.rst ├── setup.cfg ├── scripts ├── build_setup.py └── limbo.py ├── CONTRIBUTING.rst └── setup.py /requirements.devel.txt: -------------------------------------------------------------------------------- 1 | tox==3.28.0 2 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /requirements.lint.txt: -------------------------------------------------------------------------------- 1 | flake8==3.7.9 2 | pycodestyle==2.5.0 3 | pyflakes==2.1.1 4 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | Release History 2 | =============== 3 | 4 | .. include:: ../HISTORY.rst 5 | -------------------------------------------------------------------------------- /requirements.docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx==5.3.0;python_version=="3.6" 2 | Sphinx==7.0.1;python_version>"3.6" 3 | -------------------------------------------------------------------------------- /dependencies.txt: -------------------------------------------------------------------------------- 1 | build-essential 2 | libpq-dev 3 | postgresql 4 | python3-dev 5 | redis-server 6 | virtualenv 7 | -------------------------------------------------------------------------------- /TODO.rst: -------------------------------------------------------------------------------- 1 | 2 | * expose @private request decorator to users 3 | * raven integration 4 | * pipeline statsd per request 5 | -------------------------------------------------------------------------------- /talisker/logstash/talisker.pattern: -------------------------------------------------------------------------------- 1 | MODULE [0-9a-zA-Z_]+(?:\.[0-9a-zA-Z_]+)* 2 | OPTIONAL_REST_OF_LINE (?:[^\n]*) 3 | ESCAPED_QUOTES (?:[^"\\]|\\.)* 4 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.21.5 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.cfg] 7 | 8 | [bumpversion:file:talisker/__init__.py] 9 | 10 | [bumpversion:file:docs/conf.py] 11 | -------------------------------------------------------------------------------- /docs/config.rst: -------------------------------------------------------------------------------- 1 | 2 | Configuration 3 | ------------- 4 | 5 | Talisker can be configured by the environment variables listed below. These 6 | variables can also be supplied in a python file, although environment variables 7 | override any file configuration. 8 | 9 | 10 | .. talisker_config:: 11 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py 2 | include setup.cfg 3 | include README.rst AUTHORS.rst CONTRIBUTING.rst HISTORY.rst LICENSE 4 | recursive-include docs *.rst 5 | recursive-exclude docs * 6 | recursive-exclude tests * 7 | recursive-exclude scripts * 8 | recursive-include talisker/logstash *.filter *.pattern 9 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | * Simon Davy 6 | * Maximiliano Bertacchini 7 | * Adam Collard 8 | * Tom Wardill 9 | * Guillermo Gonzalez 10 | * Robin Winslow 11 | * Wouter van Bommel -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .cache 3 | .tox 4 | *.sw? 5 | *.egg-info 6 | .eggs 7 | *.egg 8 | dist 9 | build 10 | docs/_build 11 | docs/_static 12 | docs/talisker.rst 13 | docs/modules.rst 14 | env 15 | lib 16 | .coverage* 17 | htmlcov 18 | .py2env 19 | .pytest_cache 20 | log 21 | log.* 22 | tests/django_app/django_app/static 23 | tests/requirements.limbo.txt 24 | talisker/logstash/test-config 25 | talisker/logstash/test-results 26 | .pytest_cache 27 | -------------------------------------------------------------------------------- /tests/access.log: -------------------------------------------------------------------------------- 1 | 127.0.0.1 - - [05/Oct/2016:10:11:27.856 +0000] "GET / HTTP/1.1" 404 1556 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0" duration=1298 request_id=24824316-d571-4867-be15-8139e817eb04 2 | 127.0.0.1 - - [05/Oct/2016:10:11:37.124 +0000] "GET / HTTP/1.1" 404 1583 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0" duration=819 request_id=63c8024b-8761-4703-80dd-0f01453c0c46 3 | 127.0.0.1 - - [05/Oct/2016:10:11:43.961 +0000] "GET / HTTP/1.1" 404 1583 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0" duration=887 request_id=d8b07a48-7048-4802-80aa-1c49d0d6cc54 4 | -------------------------------------------------------------------------------- /tests/test_truncate.log: -------------------------------------------------------------------------------- 1 | 2016-01-13 10:24:07.357Z INFO app "msg 5 6 7 8 9 0 XXXXXX" 2 | 2016-01-13 10:24:07.357Z INFO app "msg" foo=" 5 6 7 8 9 0 XXXXXX" 3 | 2016-01-13 10:24:07.357Z INFO app "msg" 4 | 5 6 5 | 7 8 6 | 9 0 XXXXX 7 | 2016-01-13 10:24:07.357Z INFO app "msg" foo=bar 8 | 5 6 9 | 7 8 10 | 9 0 XXXXX 11 | 2016-01-13 10:24:07.357Z INFO app "msg ... 5 6 7 8 9 0 XXXXXX" 12 | 2016-01-13 10:24:07.357Z INFO app "msg ... msg2" foo=bar 13 | -------------------------------------------------------------------------------- /HACKING: -------------------------------------------------------------------------------- 1 | # Developing on Talisker 2 | 3 | 4 | First ensure dependencies are installed. For ubuntu/debian, the required 5 | packages are in dependencies.txt: 6 | 7 | cat dependencies.txt | xargs sudo apt install -y 8 | 9 | You also need to setup a db user accounts: 10 | 11 | sudo -u postgres make db-setup 12 | 13 | Now you should be able to run the tests: 14 | 15 | make test 16 | 17 | There are various test applications included too: 18 | 19 | make run # basic wsgi app 20 | make flask # flask app with db 21 | make django # django app with db 22 | make celery-{worker,client} # celery app, run each one in separate terminal 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2021 Canonical, Ltd. 2 | 3 | Licensed to the Apache Software Foundation (ASF) under one or more contributor 4 | license agreements. See the NOTICE file distributed with this work for 5 | additional information regarding copyright ownership. The ASF licenses this 6 | file to you under the Apache License, Version 2.0 (the "License"); you may not 7 | use this file except in compliance with the License. You may obtain a copy of 8 | the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software distributed 13 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 14 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations under the License. 16 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. talisker documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to talisker's documentation! 7 | ====================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | overview 16 | development 17 | context 18 | testing 19 | config 20 | logging 21 | gunicorn 22 | sentry 23 | endpoints 24 | timeouts 25 | requests 26 | statsd 27 | prometheus 28 | celery 29 | flask 30 | django 31 | postgresql 32 | contributing 33 | authors 34 | history 35 | 36 | Indices and tables 37 | ================== 38 | 39 | * :ref:`genindex` 40 | * :ref:`modindex` 41 | * :ref:`search` 42 | 43 | -------------------------------------------------------------------------------- /tests/test.log: -------------------------------------------------------------------------------- 1 | 2016-01-13 10:24:07.357Z INFO app "something happened" 2 | # module names 3 | 2016-01-13 10:24:07.357Z INFO app.module "something happened" 4 | 2016-01-13 10:24:07.357Z INFO app.other_module "something happened" 5 | # message with escapes 6 | 2016-01-13 10:24:07.357Z INFO app "something \"happened\"" 7 | # all the logfmt variations 8 | 2016-01-13 10:24:07.357Z INFO app "something happened" quoted="quoted =" unquoted=unquoted escaped="\"escaped\"" novalue empty= int=43 float=1.234 9 | # multiline traceback/trailer 10 | 2016-01-13 10:24:07.357Z INFO app "something happened" quoted="quoted =" unquoted=unquoted escaped="\"escaped\"" novalue empty= int=43 float=1.234 11 | Traceback: 12 | blah 13 | Exception 14 | # regression test for _ in module name 15 | 2017-04-24 09:38:04.512Z INFO click_package_index.gateway.base "combined_packages_metadata: requested 0 items ([]), obtained 0 public results." request_id=b62b1ce7-5981-4cd1-9102-e71075a3c708 16 | -------------------------------------------------------------------------------- /tests/django_app/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | # -*- coding: utf-8 -*- 26 | -------------------------------------------------------------------------------- /tests/django_app/django_app/management/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | -------------------------------------------------------------------------------- /tests/django_app/django_app/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | -------------------------------------------------------------------------------- /tests/django_app/django_app/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | -------------------------------------------------------------------------------- /talisker/__main__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import talisker 26 | 27 | if __name__ == '__main__': 28 | talisker.run() 29 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-20.04 10 | tools: 11 | python: "3.10" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Build documentation in the "docs/" directory with Sphinx 18 | sphinx: 19 | configuration: docs/conf.py 20 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 21 | # builder: "dirhtml" 22 | # Fail on all warnings to avoid broken references 23 | # fail_on_warning: true 24 | 25 | # Optionally build your docs in additional formats such as PDF and ePub 26 | # formats: 27 | # - pdf 28 | # - epub 29 | 30 | # Optional but recommended, declare the Python requirements required 31 | # to build your documentation 32 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 33 | python: 34 | install: 35 | - requirements: requirements.docs.txt 36 | -------------------------------------------------------------------------------- /tests/django_app/django_app/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | # This will make sure the app is always imported when 26 | # Django starts so that shared_task will use this app. 27 | from .celery import app as celery_app 28 | 29 | __all__ = ['celery_app'] 30 | -------------------------------------------------------------------------------- /tests/py36_async_tls.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | import requests 25 | import flask 26 | 27 | app = flask.Flask(__name__) 28 | 29 | 30 | @app.route('/') 31 | def home(): 32 | r = requests.get("https://httpbin.org/get") 33 | r.raise_for_status() 34 | return 'OK' 35 | -------------------------------------------------------------------------------- /.github/workflows/tox.yml: -------------------------------------------------------------------------------- 1 | name: Run tox 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - master 8 | pull_request: 9 | branches: 10 | - main 11 | - master 12 | 13 | jobs: 14 | build: 15 | name: "Project tests via tox" 16 | services: 17 | postgres: 18 | image: postgres 19 | env: 20 | POSTGRES_USER: django_app 21 | POSTGRES_PASSWORD: django_app 22 | ports: 23 | - 5432:5432 24 | runs-on: ubuntu-20.04 25 | strategy: 26 | matrix: 27 | python: [3.5, 3.6, 3.8, "3.10", "3.12"] 28 | 29 | steps: 30 | - uses: actions/checkout@v2 31 | - name: Setup Python 32 | uses: actions/setup-python@v2 33 | with: 34 | python-version: ${{ matrix.python }} 35 | env: 36 | PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" 37 | - name: Install dependencies 38 | run: | 39 | sudo apt update 40 | cat dependencies.txt | xargs sudo apt install -y 41 | python -m pip install --upgrade pip 42 | pip install 'tox<4' 'tox-gh-actions<3' 43 | - name: Start redis 44 | run: sudo systemctl start redis-server 45 | - name: Setup tox, dependencies, and run tests 46 | run: make github-tox 47 | -------------------------------------------------------------------------------- /tests/udpecho.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | import socket 25 | import sys 26 | 27 | PORT = 8125 28 | BUFSIZE = 512 29 | 30 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 31 | sock.bind(('', PORT)) 32 | while 1: 33 | data, addr = sock.recvfrom(BUFSIZE) 34 | sys.stderr.buffer.write(data + b'\n') 35 | sys.stderr.buffer.flush() 36 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | # configure updates globally 2 | # default: all 3 | # allowed: all, insecure, False 4 | update: all 5 | 6 | # configure dependency pinning globally 7 | # default: True 8 | # allowed: True, False 9 | pin: False 10 | 11 | # set the default branch 12 | # default: empty, the default branch on GitHub 13 | # branch: dev 14 | 15 | # update schedule 16 | # default: empty 17 | # allowed: "every day", "every week", .. 18 | schedule: "every week" 19 | 20 | # search for requirement files 21 | # default: True 22 | # allowed: True, False 23 | search: True 24 | 25 | # Specify requirement files by hand, default is empty 26 | # default: empty 27 | # allowed: list 28 | requirements: 29 | - setup.cfg: 30 | pin: False 31 | 32 | # add a label to pull requests, default is not set 33 | # requires private repo permissions, even on public repos 34 | # default: empty 35 | #label_prs: update 36 | 37 | # assign users to pull requests, default is not set 38 | # requires private repo permissions, even on public repos 39 | # default: empty 40 | assignees: 41 | - bloodearnest 42 | 43 | # configure the branch prefix the bot is using 44 | # default: pyup- 45 | # branch_prefix: pyup- 46 | 47 | # set a global prefix for PRs 48 | # default: empty 49 | # pr_prefix: "Bug #12345" 50 | 51 | # allow to close stale PRs 52 | # default: True 53 | # close_prs: True 54 | -------------------------------------------------------------------------------- /tests/django_app/django_app/management/commands/test_command.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | from django.core.management.base import BaseCommand, CommandError 25 | import logging 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | class Command(BaseCommand): 30 | help = 'test command' 31 | 32 | def handle(self, *args, **options): 33 | logger.info('test', extra={'foo': 'bar'}) 34 | -------------------------------------------------------------------------------- /docs/statsd.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ====== 5 | Statsd 6 | ====== 7 | 8 | Talisker provides statsd integration and configuration. 9 | 10 | Configuration 11 | ------------- 12 | 13 | Statsd can be configured by the STATSD_DSN envvar, patterned after the SENTRY_DSN. 14 | This combines all statsd config into a single DSN url. For example:: 15 | 16 | .. code-block:: bash 17 | 18 | # talk udp on port 1234 to host statsd, using a prefix of 'my.prefix' 19 | STATSD_DSN=udp://statsd:1234/my.prefix 20 | 21 | # can also use / for prefix separators, the / converted to . 22 | STATSD_DSN=udp://statsd:1234/my/prefix 23 | 24 | # ipv6 25 | STATSD_DSN=udp6://statsd:1234/my.prefix 26 | 27 | # custom max udp size of 1024 28 | STATSD_DSN=udp://statsd:1234/my.prefix?maxudpsize=1024 29 | 30 | Currently, only the udp statsd client is supported. If no config is 31 | provided, a dummy client is used that does nothing. 32 | 33 | TODO: contribute this to upstream statsd module 34 | 35 | Integration 36 | ----------- 37 | 38 | If statsd is configured, talisker will configure 39 | `gunicorn's statsd `_ 40 | functionality to use it. Additionally, it will enable statsd metrics for 41 | talisker's requests sessions. 42 | 43 | Your app code can get a statsd client by simply calling::: 44 | 45 | statsd = talisker.statsd.get_client() 46 | -------------------------------------------------------------------------------- /docs/prometheus.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ========== 5 | Prometheus 6 | ========== 7 | 8 | Talisker provides optional `prometheus_client` integration and configuration. 9 | 10 | Installation 11 | ------------ 12 | 13 | The package supports extras args to install `prometheus_client`: 14 | 15 | .. code-block:: bash 16 | 17 | $ pip install talisker[prometheus] 18 | 19 | Configuration 20 | ------------- 21 | 22 | ``prometheus_client`` integration has extensive support for 23 | multiprocessing with gunicorn. 24 | 25 | If you are only using one worker process, then regular single process 26 | mode is used. 27 | 28 | However, if you have multiple workers, then the 29 | ``prometheus_multiproc_dir`` envvar is set to a tmpdir, as per 30 | `the prometheus_client multiprocessing docs `_. 31 | This allows any worker being scraped to report metrics for all workers. 32 | 33 | However, by default it leaks mmaped files when workers are killed, 34 | wasting disk space and slowing down metric collection. Talisker provides 35 | a non-trivial workaround for this, by having the gunicorn master merge 36 | left over metrics into a single file. 37 | 38 | Note that in multiprocss mode, due to prometheus_client's design, all 39 | registered metrics are exposed, regardless of registry. 40 | 41 | The metrics are exposed at ``/_status/metrics`` 42 | -------------------------------------------------------------------------------- /tests/logstash-talisker.yaml: -------------------------------------------------------------------------------- 1 | type: talisker_access 2 | filter: talisker/logstash/talisker_access.filter 3 | tests: 4 | simple_combinedlog: 5 | input: '127.0.0.1 user auth [01/Jan/2016:12:34:56.789 +0000] "GET / HTTP/1.1" 200 1000 "referrer" "user agent"' 6 | expected: 7 | "@timestamp": "2016-01-01T12:34:56.789Z" 8 | clientip: 2fb28fa0cbae7926332c209f2f3868e7674c7689 9 | ident: "-" 10 | auth: "-" 11 | verb: "GET" 12 | request: "/" 13 | httpversion: "1.1" 14 | response: 404 15 | bytes: 1000 16 | referrer: "-" 17 | agent: "user agent" 18 | levelname: "INFO" 19 | combinedlog_with_logfmt: 20 | input: '127.0.0.1 user auth [01/Jan/2016:12:34:56.789 +0000] "GET / HTTP/1.1" 200 1000 "referrer" "user agent" duration=1000 request_id=xxx' 21 | expected: 22 | "@timestamp": "2016-01-01T12:34:56.789Z" 23 | clientip: 2fb28fa0cbae7926332c209f2f3868e7674c7689 24 | ident: "-" 25 | auth: "-" 26 | verb: "GET" 27 | request: "/" 28 | httpversion: "1.1" 29 | response: 404 30 | bytes: 1000 31 | referrer: "-" 32 | agent: "user agent" 33 | levelname: "INFO" 34 | duration: "1000" 35 | request_id: "xxx" 36 | -------------------------------------------------------------------------------- /tests/django_app/django_app/migrations/0001_create_procedure.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | from django.db import migrations 25 | 26 | 27 | sql = """ 28 | CREATE OR REPLACE FUNCTION add(integer, integer) RETURNS integer 29 | AS 'select $1 + $2;' 30 | LANGUAGE SQL 31 | IMMUTABLE 32 | RETURNS NULL ON NULL INPUT; 33 | """ 34 | 35 | 36 | class Migration(migrations.Migration): 37 | dependencies = [] 38 | operations = [migrations.RunSQL(sql)] 39 | -------------------------------------------------------------------------------- /tests/django_app/django_app/wsgi.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | """ 25 | WSGI config for django_app project. 26 | 27 | It exposes the WSGI callable as a module-level variable named ``application``. 28 | 29 | For more information on this file, see 30 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 31 | """ 32 | 33 | import os 34 | 35 | from django.core.wsgi import get_wsgi_application 36 | 37 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_app.settings") 38 | 39 | application = get_wsgi_application() 40 | -------------------------------------------------------------------------------- /requirements.tests.txt: -------------------------------------------------------------------------------- 1 | # these are pinned for repeatability, but updated by pyup.io to be latest 2 | pytest==4.6.9;python_version<"3.10" 3 | pytest>7;python_version>="3.10" 4 | more-itertools<8.11.0;python_version<"3.6" 5 | more-itertools;python_version>="3.6" 6 | freezegun==0.3.14 7 | pytest-cov==2.8.1;python_version<"3.10" 8 | pytest-cov>2.8;python_version>="3.10" 9 | pytest-postgresql==1.4.1 # pypup: <2.0 10 | pytest-xdist==1.31.0 11 | mirakuru==1.1 # pyup: <2.0 12 | pytest-timeout==1.3.4 13 | responses==0.10.6 # pyup: <0.10.9 14 | # setuptools 45+ does do py2 15 | setuptools==44.0.0;python_version<"3.10" 16 | setuptools>64;python_version>="3.10" 17 | coverage==5.0.3;python_version<"3.10" 18 | coverage>=6;python_version>="3.10" 19 | flaky==3.7.0;python_version<"3.6" 20 | flaky==3.8.1;python_version>="3.6" 21 | 22 | # for integration tests 23 | # eventlet is pinned until https://github.com/benoitc/gunicorn/pull/2581 24 | # is fixed 25 | eventlet==0.30.2;python_version<"3.10" 26 | eventlet>0.33;python_version>="3.10" 27 | gevent==20.9.0;python_version<"3.10" 28 | gevent>21;python_version>="3.10" 29 | # ABI change, incompatible with gevent<22.10.2 30 | greenlet<2;python_version<"3.10" 31 | # the bottom pin is for limbo test runs, as latest version doesn't work with 32 | # newer celery versions 33 | redis>=3.2.0 34 | SQLAlchemy==1.3.13 35 | # newer versions of celery pin vine versions, however 36 | # older versions don't. This pin is for limbo test runs for 37 | # celery 3.x on python 2.7. 38 | vine<5.0;python_version<"3.6" 39 | -------------------------------------------------------------------------------- /tests/statsd.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | BASE="sas" 25 | METRICS="${BASE}-metrics" 26 | 27 | GRAPHITE_PORT=9080 28 | STATSD_UDP_PORT=8125 29 | 30 | 31 | case $1 in 32 | pull) 33 | docker pull hopsoft/graphite-statsd 34 | ;; 35 | create) 36 | docker create --name ${METRICS} -p ${GRAPHITE_PORT}:80 -p ${STATSD_UDP_PORT}:8125/udp hopsoft/graphite-statsd 37 | ;; 38 | start) 39 | docker start ${METRICS} 40 | ;; 41 | stop) 42 | docker stop ${METRICS} 43 | ;; 44 | clean) 45 | docker rm ${METRICS} 46 | ;; 47 | esac 48 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, py36, py38, py310, py312, docs, lint, minimal_dep_version, no-extras 3 | skip_missing_interpreters = True 4 | skipsdist = True 5 | 6 | [gh-actions] 7 | python = 8 | 3.5: py35,minimal_dep_version 9 | 3.6: py36 10 | 3.8: py38,lint,docs, no-extras 11 | 3.10: py310 12 | 3.12: py312 13 | 14 | [testenv] 15 | usedevelop = True 16 | deps = 17 | -r{toxinidir}/requirements.tests.txt 18 | {toxinidir} 19 | commands = py.test -v --cov=talisker --no-success-flaky-report 20 | extras = 21 | gunicorn 22 | raven 23 | flask 24 | django 25 | celery 26 | prometheus 27 | pg 28 | asyncio 29 | setenv = 30 | LC_ALL=C.UTF-8 31 | LANG=C.UTF-8 32 | 33 | [testenv:minimal_dep_version] 34 | skip_install = True 35 | usedevelop = False 36 | deps = 37 | -r{toxinidir}/tests/requirements.limbo.txt 38 | -e{toxinidir} 39 | basepython = python3.5 40 | 41 | [testenv:no-extras] 42 | basepython = python3.8 43 | extras = 44 | 45 | [testenv:lint] 46 | skip_install = True 47 | usedevelop = False 48 | deps = -r{toxinidir}/requirements.lint.txt 49 | commands = flake8 talisker tests 50 | basepython = python3.8 51 | 52 | [testenv:docs] 53 | deps = -r{toxinidir}/requirements.docs.txt 54 | commands = sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html 55 | basepython = python3.8 56 | 57 | [testenv:packaging] 58 | skip_install = True 59 | usedevelop = False 60 | deps = -r{toxinidir}/requirements.docs.txt 61 | check-manifest 62 | commands = 63 | check-manifest -v 64 | python setup.py check --metadata --restructuredtext --strict 65 | 66 | -------------------------------------------------------------------------------- /docs/flask.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | .. _flask: 3 | 4 | ===== 5 | Flask 6 | ===== 7 | 8 | Usage 9 | ----- 10 | 11 | Talisker provides some opt-in support for flask apps. This does a few main 12 | things currently. 13 | 14 | 1) enable sentry flask support for your app. This means you will get more 15 | information in your sentry errors, as well as being able to configure sentry 16 | via your app config as normal. 17 | 18 | 2) disable flask default app logger configuration, and just use talisker's 19 | configuration. This avoids double-logged exception messages. 20 | 21 | 3) Add X-View-Name header to each response, which helps give extra logging and 22 | metric info. 23 | 24 | To enable, you can either use a special Talisker flask app:: 25 | 26 | app = talisker.flask.TaliskerApp(__name__) 27 | 28 | or register your app with Talisker afterwards:: 29 | 30 | talisker.flask.register(app) 31 | 32 | 33 | Sentry Details 34 | -------------- 35 | 36 | Talisker integrates the flask support in `raven.contrib.flask`. See `the 37 | raven flask documentation 38 | `_ for more details. 39 | 40 | The sentry flask extension is configured to work with talisker. 41 | 42 | * `logging=False` as Talisker has already set this up. This means the 43 | other possible logging config is ignored. 44 | 45 | * `wrap_wsgi=False` as Talisker has already set this up 46 | 47 | * `register_signal=True`, which is the default 48 | 49 | If for some reason you wish to configure the flask sentry extension yourself:: 50 | 51 | talisker.flask.sentry(app, **config) 52 | 53 | This has the same api as the default `raven.contrib.flask.Sentry` object, 54 | but with the above configuration. 55 | -------------------------------------------------------------------------------- /tests/django_app/django_app/celery.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | import logging 25 | import os 26 | from celery import Celery 27 | 28 | # set the default Django settings module for the 'celery' program. 29 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_app.settings') 30 | 31 | app = Celery('django_app') 32 | 33 | # Using a string here means the worker don't have to serialize 34 | # the configuration object to child processes. 35 | # - namespace='CELERY' means all celery-related configuration keys 36 | # should have a `CELERY_` prefix. 37 | app.config_from_object('django.conf:settings') 38 | 39 | # Load task modules from all registered Django app configs. 40 | app.autodiscover_tasks('django_app') 41 | 42 | 43 | @app.task(bind=True) 44 | def debug_task(self): 45 | logging.getLogger(__name__).info( 46 | 'request', extra={'request': repr(self.request)}) 47 | -------------------------------------------------------------------------------- /docs/talisker_config.py: -------------------------------------------------------------------------------- 1 | import textwrap 2 | 3 | from docutils import nodes 4 | from docutils.parsers.rst import Directive 5 | from docutils.statemachine import ViewList 6 | from sphinx.util.nodes import nested_parse_with_titles 7 | 8 | from talisker.config import CONFIG_META 9 | 10 | 11 | class talisker_config(nodes.Structural, nodes.Element): 12 | pass 13 | 14 | 15 | def visit_config_node(self, node): 16 | pass 17 | 18 | 19 | def depart_config_node(self, node): 20 | pass 21 | 22 | 23 | def extract_docstring(docstring): 24 | first_line, _, rest = docstring.partition('\n') 25 | dedented = [first_line.strip()] 26 | if rest: 27 | dedented.append(textwrap.dedent(rest)) 28 | return '\n'.join(dedented) 29 | 30 | 31 | class ConfigDirective(Directive): 32 | 33 | def run(self): 34 | config_nodes = [] 35 | for name, (attr, doc) in CONFIG_META.items(): 36 | # add title and link 37 | id = nodes.make_id('config-' + name) 38 | node = talisker_config() 39 | section = nodes.section(ids=[id]) 40 | section += nodes.title(name, name) 41 | node += section 42 | 43 | # render docstring as ReST 44 | viewlist = ViewList() 45 | docstring = extract_docstring(doc) 46 | for i, line in enumerate(docstring.splitlines()): 47 | viewlist.append(line, 'config.py', i) 48 | doc_node = nodes.section() 49 | doc_node.document = self.state.document 50 | nested_parse_with_titles(self.state, viewlist, doc_node) 51 | node += doc_node 52 | 53 | config_nodes.append(node) 54 | 55 | return config_nodes 56 | 57 | 58 | def setup(app): 59 | app.add_node(talisker_config, html=(visit_config_node, depart_config_node)) 60 | app.add_directive('talisker_config', ConfigDirective) 61 | -------------------------------------------------------------------------------- /tests/django_app/manage.py: -------------------------------------------------------------------------------- 1 | #!../../env/bin/python 2 | # 3 | # Copyright (c) 2015-2021 Canonical, Ltd. 4 | # 5 | # This file is part of Talisker 6 | # (see http://github.com/canonical-ols/talisker). 7 | # 8 | # Licensed to the Apache Software Foundation (ASF) under one 9 | # or more contributor license agreements. See the NOTICE file 10 | # distributed with this work for additional information 11 | # regarding copyright ownership. The ASF licenses this file 12 | # to you under the Apache License, Version 2.0 (the 13 | # "License"); you may not use this file except in compliance 14 | # with the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, 19 | # software distributed under the License is distributed on an 20 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 21 | # KIND, either express or implied. See the License for the 22 | # specific language governing permissions and limitations 23 | # under the License. 24 | # 25 | import os 26 | import sys 27 | 28 | if __name__ == "__main__": 29 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_app.settings") 30 | try: 31 | from django.core.management import execute_from_command_line 32 | except ImportError: 33 | # The above import may fail for some other reason. Ensure that the 34 | # issue is really that Django is missing to avoid masking other 35 | # exceptions on Python 2. 36 | try: 37 | import django 38 | except ImportError: 39 | raise ImportError( 40 | "Couldn't import Django. Are you sure it's installed and " 41 | "available on your PYTHONPATH environment variable? Did you " 42 | "forget to activate a virtual environment?" 43 | ) 44 | raise 45 | execute_from_command_line(sys.argv) 46 | -------------------------------------------------------------------------------- /docs/celery.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ====== 5 | Celery 6 | ====== 7 | 8 | Talisker provides some optional integration with celery. If your project does 9 | not have celery installed, this will not be used. 10 | 11 | To run a Celery worker with Talisker, use the provided Celery entrypoint: 12 | 13 | .. code-block:: bash 14 | 15 | $ talisker.celery worker -A myapp 16 | 17 | 18 | Logging 19 | ------- 20 | 21 | Talisker configures Celery logging to use Talisker's logging mechanisms. It 22 | ensures that the `task_id` and `task_name` will be logged with every log 23 | message for a Celery job. 24 | 25 | Additionally, if the job is triggered by a Talisker process (e.g. a Talisker 26 | gunicorn worker) it will add the `request_id` to the logging tags for the 27 | celery job when it executes. This allows you to track tasks initiated by 28 | a specific request id. 29 | 30 | 31 | Metrics 32 | ------- 33 | 34 | Talisker will enable basic celery task metrics by default. 35 | 36 | Talisker sets up histogram metrics for 37 | 38 | - celery..enqueue (time to publish to queue) 39 | - celery..queue (time in queue) 40 | - celery..run (time to run task) 41 | 42 | And counters for 43 | 44 | - celery..count (total tasks) 45 | - celery..retry (number of retried tasks) 46 | - celery..success (number of successful tasks) 47 | - celery..failure (number of failed tasks) 48 | - celery..revoked (number of revoked tasks) 49 | 50 | Note: talisker supports celery>=3.1.0. If you need to be sure, the 51 | package supports extras args to install celery dependencies: 52 | 53 | .. code-block:: bash 54 | 55 | $ pip install talisker[celery] 56 | 57 | 58 | Sentry 59 | ------ 60 | 61 | Talisker integrates Sentry with Celery, so Celery exceptions will be 62 | reported to Sentry. This uses the standard support in raven for 63 | integrating Celery and Sentry. 64 | -------------------------------------------------------------------------------- /docs/endpoints.rst: -------------------------------------------------------------------------------- 1 | 2 | ================ 3 | Status Endpoints 4 | ================ 5 | 6 | Talisker provides a set of app-agnostic standard endpoints for your app for 7 | querying its status. This is designed so that in production you have standard 8 | ways to investigate problems, and to configure load balancer health checks and 9 | nagios checks. 10 | 11 | 12 | ``/_status/ping`` 13 | A simple check designed for use with haproxy's httpcheck option, returns 14 | 200, responds to GET, HEAD, or OPTIONS, the body content being the 15 | application's revision 16 | 17 | ``/_status/check`` 18 | For use with nagios check_http plugin, or similar. 19 | 20 | It tries to hit ``/_status/check`` in your app. If that is not found, 21 | it just returns a 200, as a basic proxy for the application being up. 22 | 23 | ``/_status/test/sentry`` (``/_status/error`` for backwards compatibility) 24 | Raise a test error, designed to test sentry/raven integration. 25 | 26 | ``/_status/test/statsd`` 27 | Send a test metric value. Designed to test statsd integration. 28 | 29 | ``/_status/test/prometheus`` 30 | Increment a test counter. Designed to test Prometheus integration. 31 | 32 | ``/_status/metrics`` 33 | Exposes prometheus metrics in Prometheus text format. 34 | 35 | ``/_status/info/config`` 36 | Shows the current Talisker configuration 37 | 38 | ``/_status/info/packages`` 39 | Shows a list of installed python packages and their versions 40 | 41 | ``/_status/info/workers`` 42 | Shows a summary of master and worker processes (e.g CPU, memory, fd count) 43 | and other process information. *Only available if psutil is installed.* 44 | 45 | ``/_status/info/objgraph`` 46 | Shows the most common python objects in user for the worker that services 47 | the request. *Only available if objgraph is installed.* 48 | 49 | ``/_status/info/logtree`` 50 | Displays the stdlib logging configuration using logging_tree. *Only 51 | available if logging_tree is installed.* 52 | 53 | -------------------------------------------------------------------------------- /tests/test_django.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import pytest 26 | 27 | try: 28 | import django # noqa 29 | except ImportError: 30 | pytest.skip("skipping django only tests", allow_module_level=True) 31 | 32 | import talisker.django 33 | import talisker.sentry 34 | from talisker.testing import TEST_SENTRY_DSN 35 | 36 | 37 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed') 38 | def test_django_sentry_client(monkeypatch, context): 39 | from talisker.sentry import DummySentryTransport 40 | called = [False] 41 | 42 | def hook(): 43 | called[0] = True 44 | 45 | monkeypatch.setattr('raven.contrib.django.client.install_sql_hook', hook) 46 | client = talisker.django.SentryClient( 47 | dsn=TEST_SENTRY_DSN, 48 | transport=DummySentryTransport, 49 | install_sql_hook=True, 50 | ) 51 | 52 | assert called[0] is False 53 | assert set(client.processors) == talisker.sentry.default_processors 54 | context.assert_log(msg='configured raven') 55 | assert talisker.sentry.get_client() is client 56 | assert talisker.sentry.get_log_handler().client is client 57 | -------------------------------------------------------------------------------- /tests/django_app/django_app/views.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | import logging 25 | from django.shortcuts import render 26 | from django.http import HttpResponse 27 | 28 | from django_app.celery import debug_task 29 | 30 | from django.db import connection 31 | from django.contrib.auth.models import User, Group 32 | 33 | 34 | def index(request): 35 | return HttpResponse('ok', status=200) 36 | 37 | 38 | def error(request): 39 | User.objects.count() 40 | Group.objects.count() 41 | User.objects.get(pk=1) 42 | 43 | with connection.cursor() as cursor: 44 | cursor.execute("select add(2, 3);") 45 | cursor.fetchone() 46 | cursor.execute("select add(%s, %s)", [2, 3]) 47 | cursor.fetchone() 48 | cursor.callproc('add', [2, 3]) 49 | cursor.fetchone() 50 | 51 | raise Exception('test') 52 | 53 | 54 | def db(request): 55 | count = User.objects.count() 56 | raise Exception('test') 57 | return HttpResponse('There are %d users' % count, status=200) 58 | 59 | 60 | def celery(request): 61 | logging.getLogger(__name__).info('starting task') 62 | debug_task.delay() 63 | return HttpResponse('ok', status=200) 64 | -------------------------------------------------------------------------------- /docs/development.rst: -------------------------------------------------------------------------------- 1 | Development using Talisker 2 | ========================== 3 | 4 | Talisker has been designed with the goal of working in development *and* 5 | production. This is to try and encourage the same tool used throughout. 6 | 7 | Talisker's default configuration is designed for production usage, e.g.: 8 | 9 | - only INFO level logs and above go to stderr 10 | - python's warning system is disabled 11 | 12 | 13 | DEVEL Mode 14 | ---------- 15 | 16 | If the DEVEL env var is set, Talisker will run in DEVEL mode. 17 | 18 | What this means varies on which tool you are using, but at a base level it 19 | enables python's warning logs, as you generally want these in development. 20 | 21 | For Gunicorn, DEVEL mode means a few more things: 22 | 23 | - sets timeout to 99999, to avoid timeouts when debugging 24 | - it enables auto reloading on code changes 25 | 26 | Also, for developer convenience, if you manually set Gunicorn's debug level to DEBUG, when 27 | in DEVEL mode, Talisker will actually log debug level messages to stderr. 28 | 29 | 30 | Development Logging 31 | ------------------- 32 | 33 | Talisker logs have been designed to be readable in development. 34 | 35 | This includes: 36 | 37 | - preserving the common first 4 fields in python logging for developer familiarity. 38 | 39 | - tags are rendered most-specific to least specific. This means that the tags 40 | a developer is interested in are likely first. 41 | 42 | - if stderr is an interactive tty, then logs are colorized, to aid human reading. 43 | 44 | 45 | Colored Output 46 | -------------- 47 | 48 | If in DEVEL mode, and stdout is a tty device, then Talisker will colorise log output. 49 | 50 | To disable this, you can set the env var: 51 | 52 | .. code-block:: bash 53 | 54 | TALISKER_COLOR=no 55 | 56 | or 57 | 58 | .. code-block:: bash 59 | 60 | TERM=dumb 61 | 62 | The colorscheme looks best on dark terminal backgrounds, but should be readable on 63 | light terminals too. 64 | 65 | If your terminal doesn't support bold, dim, or italic text formatting, it might 66 | look unpleasent. In that case, you can try the simpler colors 67 | 68 | .. code-block:: bash 69 | 70 | TALISKER_COLOR=simple 71 | 72 | -------------------------------------------------------------------------------- /docs/gunicorn.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ======== 5 | Gunicorn 6 | ======== 7 | 8 | 9 | Basic Usage 10 | ----------- 11 | 12 | Gunicorn is the wsgi server used by Talisker. To use it, simply use the Talisker 13 | wrapper in place of regular gunicorn script. 14 | 15 | .. code-block:: bash 16 | 17 | $ talisker.gunicorn -c config.py myapp:wsgi_app 18 | 19 | This wrapper simply initialises Talisker before passing control to Gunicorn's 20 | entrypoint. As such, it takes exactly the same command line arguments, and 21 | behaves in the same way. 22 | 23 | Talisker supports the sync, gevent, and eventlet workers. Others workers may 24 | work, but have not been tested. The only place it matters is in the 25 | context-local storage of Talsiker log tags and request ids. Talisker will use 26 | greenlet based contexts if it finds itself running in a greenlet context, or 27 | else a thread local object. 28 | 29 | 30 | 31 | Python 3.6, Async and Requests 32 | ------------------------------ 33 | 34 | Due to changes in the SSL api in python 3.6, requests currently has a bug with 35 | https endpoints in monkeypatched async context. The details are at 36 | ``_, but basically the 37 | monkeypatching must be done *before* requests is imported. Normally, this 38 | would not affect gunicorn, as your app would only import requests in a worker 39 | process after the monkeypatch has been applied. However, because Talisker 40 | enables some integrations in the main process, before the gunicorn code is run, 41 | it triggers this bug. Specfically, we import the raven library to get early 42 | error handling, and raven imports requests. 43 | 44 | We provide two special entrypoints to work around this problem, if you are 45 | using python 3.6 and eventlet or gevent workers with the requests library. 46 | They simply apply the appropriate monkeypatching first, before then just 47 | initialising talikser and running gunicorn as normal. 48 | 49 | .. code-block:: bash 50 | 51 | $ talisker.gunicorn.eventlet --worker-class eventlet -c config.py myapp:wsgi_app 52 | 53 | .. code-block:: bash 54 | 55 | $ talisker.gunicorn.gevent --worker-class gevent -c config.py myapp:wsgi_app 56 | 57 | -------------------------------------------------------------------------------- /tests/django_app/django_app/urls.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | """django_app URL Configuration 25 | 26 | The `urlpatterns` list routes URLs to views. For more information please see: 27 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 28 | Examples: 29 | Function views 30 | 1. Add an import: from my_app import views 31 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 32 | Class-based views 33 | 1. Add an import: from other_app.views import Home 34 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 35 | Including another URLconf 36 | 1. Import the include() function: from django.conf.urls import url, include 37 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 38 | """ 39 | from django.contrib import admin 40 | from django.urls import re_path 41 | from django_app import views 42 | 43 | from django.conf import settings 44 | from django.conf.urls.static import static 45 | 46 | urlpatterns = [ 47 | re_path(r'^admin/', admin.site.urls), 48 | re_path(r'', views.index), 49 | re_path(r'^error/', views.error), 50 | re_path(r'^celery/', views.celery), 51 | re_path(r'^db/', views.db), 52 | ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) 53 | -------------------------------------------------------------------------------- /tests/wsgi_app.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import pprint 26 | import logging 27 | import time 28 | 29 | 30 | def application(environ, start_response): 31 | if environ['PATH_INFO'] == '/_status/check': 32 | status = '404 Not Found' 33 | else: 34 | status = '200 OK' 35 | start_response(status, [('content-type', 'text/plain')]) 36 | output = pprint.pformat(environ) 37 | logger = logging.getLogger(__name__) 38 | logger.debug('debug') 39 | logger.info('info') 40 | logger.warning('warning') 41 | logger.error('error') 42 | logger.critical('critical') 43 | return [output.encode('utf8')] 44 | 45 | 46 | def app404(environ, start_response): 47 | if environ['PATH_INFO'] == '/_status/ping': 48 | start_response('200 OK', []) 49 | return [b'OK'] 50 | else: 51 | start_response('404 Not Found', []) 52 | return [b'Not Found'] 53 | 54 | 55 | def timeout(environ, start_response): 56 | time.sleep(1000) 57 | 58 | 59 | def timeout2(environ, start_response): 60 | start_response('200 OK', [('content-type', 'text/plain')]) 61 | time.sleep(1000) 62 | 63 | 64 | def timeout3(environ, start_response): 65 | start_response('200 OK', [('content-type', 'text/plain')]) 66 | 67 | def i(): 68 | yield time.sleep(1000) 69 | 70 | return i() 71 | -------------------------------------------------------------------------------- /docs/timeouts.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Request Timeouts 3 | ================ 4 | 5 | Talisker supports the idea of a request deadline, with the goal of 6 | failing early, especially when under load. This deadline can be 7 | specified as a timeout, either globaly or per-endpoint. 8 | 9 | Talisker will try to use the remaining time left until the deadline as 10 | network timeout parameters. It supports HTTP and SQL requests out of the 11 | box, if you use `talisker.requests.TaliskerAdapter` and 12 | `talisker.postgresql.TaliskerConnecton`, respectivley. It also provides 13 | an API to get the remaining time left before the deadline, which you can 14 | use in other network operations. 15 | 16 | .. code-block:: python 17 | 18 | timeout = Context.deadline_timeout() 19 | 20 | Note: this will raise `talisker.DeadlineExceeded` if the deadline has 21 | been exceeded. 22 | 23 | Talisker timeouts are not hard guarantees - Talisker will not cancel 24 | your request. They merely try to ensure that network operations will 25 | fail earlier rather than blocking for long periods. 26 | 27 | Deadline Propagation 28 | -------------------- 29 | 30 | The deadline can be set via a the X-Request-Deadline request header, as 31 | an ISO 8601 datestring. This will override the configured endpoint 32 | deadline, if any. Talisker's requests support will also send the current 33 | deadline as a header in any outgoing requests. This allows API gateway 34 | services to communicate top-level request deadlines ina calls to other 35 | services. 36 | 37 | 38 | Configuring Timeouts 39 | -------------------- 40 | 41 | You can set a global timeout via the TALISKER_REQUEST_TIMEOUT config, or 42 | per endpoint with the `talisker.request_timeout` decorator. 43 | 44 | .. code-block:: python 45 | 46 | @talisker.request_timeout(3000) # milliseconds 47 | def view(request): 48 | ... 49 | 50 | 51 | Soft Timeouts 52 | ------------- 53 | 54 | Talisker supports the concept of a `soft_timeout`, which will 55 | send a sentry report if a request takes longer than the soft timeout 56 | threshold. This is useful to provide richer information for problematic 57 | requests. 58 | 59 | You can set this global via the TALISKER_SOFT_REQUEST_TIMEOUT 60 | config or per endpoint via the `talisker.request_timeout` decorator. 61 | 62 | .. code-block:: python 63 | 64 | @talisker.request_timeout(soft_timeout=3000) # milliseconds 65 | def view(request): 66 | ... 67 | -------------------------------------------------------------------------------- /docs/postgresql.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ========== 5 | Postgresql 6 | ========== 7 | 8 | Talisker provides some optional integration with postgresql via pyscopg2, in 9 | the form of a custom connection/cursor implementation that integrates with 10 | logging and with sentry breadcrumbs. 11 | 12 | Ensure you have the correct dependencies by using the `pg` extra:: 13 | 14 | pip install talisker[pg] 15 | 16 | To use it in Django:: 17 | 18 | DATABASES = { 19 | 'default': { 20 | 'ENGINE': 'django.db.backends.postgresql', 21 | ..., 22 | 'OPTIONS': { 23 | 'connection_factory': talisker.postgresql.TaliskerConnection 24 | } 25 | } 26 | } 27 | 28 | To use with sqlalchemy:: 29 | 30 | engine = sqlalchemy.create_engine( 31 | ..., 32 | connect_args={'connection_factory': talisker.postgresql.TaliskerConnection}, 33 | ) 34 | 35 | 36 | Query Security 37 | -------------- 38 | 39 | Given the security sensitive nature of raw sql queries, Talisker is very 40 | cautious about what it might communicate externally, either via logs or via 41 | sentry. 42 | 43 | Talisker will only ever log the query string with placeholders, and never the 44 | query parameters. This avoids leakage of sensitive information altogether, 45 | while still providing enough info to be useful to users trying to debug problems. 46 | If a query does not use query parameter, the query string is not sent, as there 47 | is no way to determine if it is sensitive or not. 48 | 49 | One exception to this is stored procedures with parameters. The only access to 50 | the query is via the raw query that was actually run, which has already merged 51 | the query parameters, so we never send the raw query. 52 | 53 | Note: in the future, we plan to add support for customised query sanitizing. 54 | 55 | Slow query Logging 56 | ------------------ 57 | 58 | The connection logs slow queries to the `talisker.slowqueries` logger. The 59 | default timeout is -1, which disables slow query logging, but can be controlled with the 60 | TALISKER_SLOWQUERY_THRESHOLD env var. The value is measured in milliseconds:: 61 | 62 | export TALISKER_SLOWQUERY_THRESHOLD=100 # log queries over 100ms 63 | 64 | 65 | Sentry Breadcrumbs 66 | ------------------ 67 | 68 | Talisker will capture all queries run during a request or other context as 69 | Sentry breadcrumbs. In the case of an error, they will include them in the 70 | report to sentry. 71 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | Talisker - an opinionated WSGI app platform 3 | =========================================== 4 | 5 | .. image:: https://github.com/canonical-ols/talisker/actions/workflows/tox.yml/badge.svg?branch=master 6 | :target: https://github.com/canonical-ols/talisker/actions?workflow=tox 7 | :alt: CI Status 8 | 9 | .. image:: https://readthedocs.org/projects/talisker/badge/?version=latest 10 | :target: https://readthedocs.org/projects/talisker/?badge=latest 11 | :alt: Documentation Status 12 | 13 | Talisker is an enhanced runtime for your WSGI application that aims to provide 14 | a common operational platform for your python microservices. 15 | 16 | It integrates with many standard python libraries to give you out-of-the-box 17 | logging, metrics, error reporting, status urls and more. 18 | 19 | Python version support 20 | ---------------------- 21 | 22 | Talisker 0.20.0 was the last to support Python 2.7. 23 | Talisker version >=0.21.0 only supports Python 3.x, as 24 | they come with Ubuntu LTS releases. 25 | 26 | Quick Start 27 | ----------- 28 | 29 | Simply install Talisker with Gunicorn via pip:: 30 | 31 | pip install talisker[gunicorn] 32 | 33 | And then run your WSGI app with Talisker (as if it was regular gunicorn).:: 34 | 35 | talisker.gunicorn app:wsgi -c config.py ... 36 | 37 | This gives you 80% of the benefits of Talisker: structured logging, metrics, 38 | sentry error handling, standardised status endpoints and more. 39 | 40 | Note: right now, Talisker has extensive support for running with Gunicorn, with 41 | more WSGI server support planned. 42 | 43 | 44 | Elevator Pitch 45 | -------------- 46 | 47 | Talisker integrates and configures standard python libraries into a single 48 | tool, useful in both development and production. It provides: 49 | 50 | - structured logging for stdlib logging module (with grok filter) 51 | - gunicorn as a wsgi runner 52 | - request id tracing 53 | - standard status endpoints 54 | - statsd/prometheus metrics for incoming/outgoing http requests and more. 55 | - deep sentry integration 56 | 57 | It also optionally supports the same level of logging/metrics/sentry 58 | integration for: 59 | 60 | - celery workers 61 | - general python scripts, like cron jobs or management tasks. 62 | 63 | Talisker is opinionated, and designed to be simple to use. As such, it is not 64 | currently very configurable. However, PR's are very welcome! 65 | 66 | For more information, see The Documentation, which should be found at: 67 | 68 | https://talisker.readthedocs.io 69 | -------------------------------------------------------------------------------- /docs/django.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | .. _django: 4 | 5 | ====== 6 | Django 7 | ====== 8 | 9 | Talisker provides opt-in support for Django apps. 10 | 11 | .. _django_logging: 12 | 13 | Logging 14 | ------- 15 | 16 | To integrate with Talisker, you should at a minimum disable Django's default 17 | logging in your settings.py:: 18 | 19 | LOGGING_CONFIG = None 20 | 21 | If you don't, you'll get Django's `default logging configuration 22 | `_ 23 | in addition to Talisker's, leading to some duplicated logs in 24 | development, and possibly emails of errors in production, which is often 25 | not a good idea. 26 | 27 | If you have custom logging you want to add on top of Talisker's, you can 28 | follow the Django documentation for `configuring logging yourself 29 | `_, 30 | with something like:: 31 | 32 | LOGGING = {...} 33 | LOGGING_CONFIG = None 34 | import logging.config 35 | logging.config.dictConfig(LOGGING) 36 | 37 | which is exactly what Django does, but without the default logging. 38 | 39 | Sentry 40 | ------ 41 | 42 | To integrate with Talisker's sentry support, add raven to INSTALLED_APPS 43 | as normal, and also set SENTRY_CLIENT in your settings.py:: 44 | 45 | INSTALLED_APPS = [ 46 | 'raven.contrib.django.raven_compat', 47 | ... 48 | ] 49 | SENTRY_CLIENT = 'talisker.django.SentryClient' 50 | 51 | 52 | This will ensure the extra info Talsiker adds to Sentry messages will be 53 | applied, and also that the WSGI and logging handlers will use your Sentry 54 | configuration in settings.py. It will also set `install_sql_hook=False`, as 55 | that leaks raw SQL to the sentry server for every query. This will 56 | hopefully be addressed in a future release. 57 | 58 | 59 | Metadata 60 | -------- 61 | 62 | Talisker supports the use of an X-View-Name header for better introspection. This 63 | is used for metric and logging information, to help debugging. 64 | 65 | To support this in Django, simply add the following middleware, in any order:: 66 | 67 | MIDDLEWARE = [ 68 | ... 69 | 'talisker.django.middleware', 70 | ] 71 | 72 | 73 | Management Tasks 74 | ---------------- 75 | 76 | If you use management tasks, and want them to run with Talisker logging, 77 | you can use the generic talisker runner: 78 | 79 | .. code-block:: bash 80 | 81 | talisker.run manage.py ... 82 | 83 | or 84 | 85 | .. code-block:: bash 86 | 87 | python -m talisker manage.py ... 88 | -------------------------------------------------------------------------------- /docs/context.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Request Context 3 | =============== 4 | 5 | Talisker uses an implicit context to track requests during execution. 6 | It does this via the contextvars module from the Python standard library 7 | in Python 3.7+, falling back to the contextvars backport from PyPI. It 8 | also includes a minimal backport of ContextVar for use with Python 2. 9 | 10 | Talisker creates a new context for each WSGI request or Celery job, and 11 | tracks the request id and other data in that context, such as timeout data. 12 | Asyncio is supported, either by the native support in python 3.7, or via the 13 | aiocontextvars package, which you can install by using the asyncio 14 | extra:: 15 | 16 | pip install talisker[asyncio] 17 | 18 | Note: you need at least python 3.5.3+ to use asyncio with contextvars 19 | - aiocontextvars does not work on earlier versions. 20 | 21 | Talisker also explicitly supports contexts when using the Gevent or 22 | Eventlet Gunicorn workers, by swapping the thread local storage out for 23 | the relative greenlet based storage. This support currently does not 24 | work in python 3.7 or above, as it is not possible to switch the 25 | underlying storage in the stdlib version of the contextvars library. 26 | 27 | 28 | Request Id 29 | ---------- 30 | 31 | One of the key elements of the context is to track the current request 32 | id. This id can be supplied via a the X-Request-Id header, or else 33 | a uuid4 is used. 34 | 35 | This id is automatically attached to all log messages emitted during the 36 | request, as well as the detailed log message talisker emits for the 37 | request itself. 38 | 39 | Talisker also support propagating this request id wherever possible. 40 | When using Talisker's requests support, the current request id will be 41 | included in the outgoing request headers. When queuing celery jobs, the 42 | current request id will be passed as a header for that job, and then 43 | used by the job for all log messages when the job runs. 44 | 45 | This allows deep tracing of a particular request id across multiple 46 | services boundaries, which is key to debugging complex issues in 47 | distributed systems. 48 | 49 | 50 | Context API 51 | ----------- 52 | 53 | Talisker exposes a public API for the current context. 54 | 55 | .. code-block:: python 56 | 57 | from talisker import Context 58 | 59 | Context.request_id # get/set current request id 60 | Context.clear() # clear the current context 61 | Context.new() # create a new context 62 | 63 | # you can also add extras to the current logging context 64 | 65 | Context.logging.push(foo=1) 66 | 67 | # or 68 | 69 | with Context.logging(bar=2): 70 | ... 71 | -------------------------------------------------------------------------------- /docs/sentry.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ====== 5 | Sentry 6 | ====== 7 | 8 | Talisker provides out-of-the-box integration with sentry. 9 | 10 | Specifically, Talisker adds: 11 | 12 | * some default configuration of the sentry client 13 | * handle WSGI errors 14 | * sentry error log handler (for logged exception messages) 15 | * log message, sql and http call breadcrumbs 16 | * sentry integration with flask, django, and celery 17 | 18 | To get the current sentry client, simply use:: 19 | 20 | talisker.sentry.get_client() 21 | 22 | Error Data 23 | ---------- 24 | 25 | Talisker configures sentry breadcrumbs for log messages at INFO or higher level. 26 | It also adds the request id as a tag to the sentry message. 27 | 28 | If you want to add some custom error context, you can use the client above as 29 | you would use the sentry client as normal. For example:: 30 | 31 | client = talisker.sentry.get_client() 32 | client.context.merge({'tags': my_tags}) 33 | 34 | 35 | Sentry Configuration 36 | -------------------- 37 | 38 | Talisker uses the default SENTRY_DSN env var to configure sentry by 39 | default. Simply setting this will enable sentry for wsgi and logging. 40 | 41 | In addition, Talisker configures the sentry client by default as follows: 42 | 43 | - sets `install_logging_hook=False`, as Talisker handles it 44 | - sets `release` to the current revision 45 | - sets `hook_libraries=[]`, disabling breadcrumbs for request/httplib 46 | - sets `environment` to TALISKER_ENV envvar 47 | - sets `name` to TALISKER_UNIT envvar 48 | - sets `site` to TALISKER_DOMAIN envvar 49 | - ensures the RemovePostDataProcessor, SanitizePasswordsProcessor, and 50 | RemoveStackLocalsProcessor processors are always included, to be safe by 51 | default. 52 | 53 | 54 | If you are using Talisker's :ref:`flask` or :ref:`django` integration, you can configure 55 | your sentry client further via the usual config methods for those frameworks. 56 | 57 | If you wish to manually configure the sentry client, use the following:: 58 | 59 | talisker.sentry.configure_client(**config) 60 | 61 | This will reconfigure and reset the sentry client used by the wsgi middleware 62 | and logging handler that Talisker sets up. 63 | 64 | If you want to set your own client instance, do:: 65 | 66 | talisker.sentry.set_client(client) 67 | 68 | Whichever way you wish to configure sentry, talisker will honour your 69 | configuration except for 2 things 70 | 71 | 1) `install_logging_hook` will always be set to false, or else you'll get 72 | duplicate exceptions logged to sentry. 73 | 74 | 2) the processors will always include the base 3, although you can add more. 75 | If you really need to remove the default processors, you can modify the 76 | list at `talisker.sentry.default_processors` and then call 77 | `talisker.sentry.set_client()`. 78 | -------------------------------------------------------------------------------- /tests/flask_app.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | import logging 25 | 26 | from flask import Flask 27 | import sqlalchemy 28 | from sqlalchemy import Table, Column, Integer, String, MetaData, select 29 | from werkzeug.wrappers import Response 30 | 31 | import talisker.flask 32 | from talisker.postgresql import TaliskerConnection 33 | 34 | logger = logging.getLogger(__name__) 35 | engine = sqlalchemy.create_engine( 36 | 'postgresql://django_app:django_app@localhost:5432/django_app', 37 | connect_args={'connection_factory': TaliskerConnection}, 38 | ) 39 | 40 | metadata = MetaData() 41 | users = Table( 42 | 'users', 43 | metadata, 44 | Column('id', Integer, primary_key=True), 45 | Column('name', String), 46 | Column('fullname', String), 47 | ) 48 | 49 | metadata.create_all(engine) 50 | conn = engine.connect() 51 | conn.execute(users.insert().values(name='jack', fullname='Jack Jones')) 52 | 53 | 54 | app = Flask(__name__) 55 | talisker.flask.register(app) 56 | logger = logging.getLogger(__name__) 57 | 58 | 59 | @app.route('/') 60 | def index(): 61 | return 'ok' 62 | 63 | 64 | @app.route('/logging') 65 | def logging(): 66 | logger.info('info', extra={'foo': 'bar'}) 67 | app.logger.info('app logger') 68 | talisker.requests.get_session().post( 69 | 'http://httpbin.org/post', json={'foo': 'bar'}) 70 | return 'ok' 71 | 72 | 73 | @app.route('/error/') 74 | def error(): 75 | conn.execute(select([users]).where(users.c.id == 1)) 76 | talisker.requests.get_session().post( 77 | 'http://httpbin.org/post', json={'foo': 'bar'}) 78 | logger.info('halp', extra={'foo': 'bar'}) 79 | raise Exception('test') 80 | 81 | 82 | @app.route('/nested') 83 | def nested(): 84 | logger.info('here') 85 | resp = talisker.requests.get_session().get('http://10.0.4.1:1234') 86 | return Response(resp.content, status=200, headers=resp.headers.items()) 87 | -------------------------------------------------------------------------------- /tests/celery_app.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import logging 26 | import celery 27 | import talisker.requests 28 | import talisker.testing 29 | 30 | app = celery.Celery( 31 | 'tests.celery_app', 32 | broker='redis://localhost:6379', 33 | backend='redis://localhost:6379', 34 | task_serializer='json', 35 | ) 36 | logger = logging.getLogger(__name__) 37 | 38 | 39 | @app.task(bind=True) 40 | def basic_task(self): 41 | logger.info('basic task') 42 | return 'basic' 43 | 44 | 45 | @app.task(bind=True) 46 | def error_task(self): 47 | logger.info('error task') 48 | try: 49 | raise Exception('failed task') 50 | except Exception: 51 | self.retry(countdown=1, max_retries=1) 52 | 53 | 54 | @app.task 55 | def propagate_task(): 56 | logger.info('propagate_task') 57 | secondary_task.delay() 58 | 59 | 60 | @app.task 61 | def secondary_task(): 62 | logger.info('secondary_task') 63 | 64 | import responses 65 | with responses.RequestsMock() as rsps: 66 | rsps.add('GET', 'http://example.com') 67 | talisker.requests.get_session().get('http://example.com') 68 | logger.info('request headers', extra=rsps.calls[0].request.headers) 69 | 70 | 71 | if __name__ == '__main__': 72 | import talisker 73 | talisker.initialise() 74 | import talisker.celery 75 | talisker.celery.enable_signals() 76 | logger = logging.getLogger('tests.celery_app') 77 | logger.info('starting') 78 | basic_task.delay() 79 | logger.info('started job a') 80 | with talisker.testing.request_id('a'): 81 | basic_task.delay() 82 | logger.info('started job a with id a') 83 | basic_task.delay() 84 | logger.info('started job a') 85 | with talisker.testing.request_id('b'): 86 | error_task.delay() 87 | logger.info('started job b with id b') 88 | with talisker.testing.request_id('c'): 89 | job = error_task.delay() 90 | logger.info('started job b with id c') 91 | job.revoke() 92 | logger.info('revoked job b') 93 | error_task.apply() 94 | with talisker.testing.request_id('d'): 95 | propagate_task.delay() 96 | logger.info('done') 97 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test = pytest 3 | 4 | [flake8] 5 | exclude = .*,env,lib,dist,build,tests/django_app 6 | ignore = E402,W503 7 | 8 | [tool:pytest] 9 | testpaths = tests docs 10 | norecursedirs = django_app 11 | # pytest-postgresql gets confused in travis env, which also has pg 10 installed 12 | postgresql_exec = /usr/lib/postgresql/9.5/bin/pg_ctl 13 | filterwarnings = ignore 14 | 15 | [metadata] 16 | name = talisker 17 | version = 0.21.5 18 | description = A common WSGI stack 19 | long_description = file: README.rst 20 | author = Simon Davy 21 | author_email = simon.davy@canonical.com 22 | url = https://github.com/canonical-ols/talisker 23 | keywords = talisker 24 | classifiers = 25 | License :: OSI Approved :: Apache Software License 26 | Development Status :: 4 - Beta 27 | Intended Audience :: Developers 28 | Natural Language :: English 29 | Topic :: Internet :: WWW/HTTP :: WSGI 30 | Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware 31 | Topic :: System :: Logging 32 | Programming Language :: Python :: 3.5 33 | Programming Language :: Python :: 3.6 34 | Programming Language :: Python :: 3.8 35 | Programming Language :: Python :: 3.10 36 | Programming Language :: Python :: 3.12 37 | Programming Language :: Python :: Implementation :: CPython 38 | 39 | [options] 40 | zip_safe = False 41 | include_package_data = True 42 | packages = talisker 43 | test_suite = tests 44 | package_dir = talisker=talisker 45 | install_requires = 46 | Werkzeug~=1.0;python_version~="3.5.0" 47 | Werkzeug<3;python_version>="3.6" 48 | statsd~=3.3;python_version~="3.5.0" 49 | statsd<4;python_version>="3.6" 50 | requests~=2.25;python_version~="3.5.0" 51 | requests<3.0;python_version>"3.5" 52 | contextvars~=2.4;python_version>="3.5" and python_version<"3.7" 53 | 54 | [options.extras_require] 55 | gunicorn = 56 | gunicorn>=19.7.0;python_version>"3.6" 57 | gunicorn==19.7.0,<21.0;python_version>="3.5" and python_version<"3.8" 58 | raven = raven>=6.4.0 59 | celery = 60 | celery~=4.4;python_version~="3.5.0" 61 | celery>=4;python_version>"3.5" 62 | django = 63 | django~=2.2;python_version~="3.5.0" 64 | django<5;python_version>"3.5" 65 | prometheus = 66 | prometheus-client~=0.7.0;python_version~="3.5.0" 67 | prometheus-client<0.8;python_version>"3.5" 68 | flask = 69 | flask~=1.1;python_version~="3.5.0" 70 | flask<3;python_version>"3.5" 71 | blinker~=1.5;python_version~="3.5.0" 72 | blinker<2;python_version>"3.5" 73 | dev = 74 | logging_tree>=1.9 75 | pygments>=2.11 76 | psutil>=5.9 77 | objgraph>=3.5 78 | pg = 79 | sqlparse>=0.4.2 80 | psycopg2>=2.8,<3.0 81 | asyncio = 82 | aiocontextvars==0.2.2;python_version>="3.5.3" and python_version<"3.7" 83 | gevent = gevent>=20.9.0 84 | 85 | [options.package_data] 86 | talisker = logstash/* 87 | 88 | [options.entry_points] 89 | console_scripts = 90 | talisker=talisker:run_gunicorn[gunicorn] 91 | talisker.run=talisker:run 92 | talisker.gunicorn=talisker:run_gunicorn[gunicorn] 93 | talisker.gunicorn.eventlet=talisker:run_gunicorn_eventlet[gunicorn] 94 | talisker.gunicorn.gevent=talisker:run_gunicorn_gevent[gunicorn] 95 | talisker.celery=talisker:run_celery[celery] 96 | talisker.help=talisker:run_help 97 | -------------------------------------------------------------------------------- /talisker/django.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import logging 26 | 27 | import talisker.sentry 28 | 29 | 30 | # raven's django support does some very odd things. 31 | # There is a module global, that is a proxy to the output of 32 | # raven.contrib.django.models.get_client() 33 | # But the only way you can customise the set up is via subclassing the client. 34 | # So that's what we do. We ensure talisker's configuration, and hook it in to 35 | # the other things that need to know about the client. Django users just need 36 | # to add the following to settings: 37 | # SENTRY_CLIENT = 'talisker.django.SentryClient' 38 | 39 | if talisker.sentry.enabled: 40 | from raven.contrib.django.client import DjangoClient 41 | 42 | class SentryClient(DjangoClient): 43 | def __init__(self, *args, **kwargs): 44 | # SQL hook sends raw SQL to the server. Not cool, bro. 45 | kwargs['install_sql_hook'] = False 46 | talisker.sentry.ensure_talisker_config(kwargs) 47 | logging.getLogger(__name__).info( 48 | 'updating raven config from django app') 49 | super().__init__(*args, **kwargs) 50 | # update any previously configured sentry client 51 | talisker.sentry.set_client(self) 52 | 53 | def build_msg(self, event_type, *args, **kwargs): 54 | data = super().build_msg(event_type, *args, **kwargs) 55 | talisker.sentry.add_talisker_context(data) 56 | return data 57 | 58 | def set_dsn(self, dsn=None, transport=None): 59 | super().set_dsn(dsn, transport) 60 | talisker.sentry.log_client(self) 61 | 62 | else: 63 | 64 | class SentryClient(): 65 | def __init__(self, *args, **kwargs): 66 | raise Exception('raven is not installed') 67 | 68 | 69 | def middleware(get_response): 70 | """Set up middleware to add X-View-Name header.""" 71 | def add_view_name(request): 72 | response = get_response(request) 73 | if getattr(request, 'resolver_match', None): 74 | view_name = request.resolver_match.view_name 75 | response['X-View-Name'] = view_name 76 | request.environ['VIEW_NAME'] = view_name 77 | return response 78 | return add_view_name 79 | -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | ======== 4 | Overview 5 | ======== 6 | 7 | 8 | Goals 9 | ----- 10 | 11 | Talisker is designed with the following high level goals: 12 | 13 | * to provide a standard platform for running wsgi apps 14 | * to support both operational and developer workflows in the same tool 15 | * to be easy as possible to integrate with any wsgi app and framework 16 | 17 | The original motivation was to standardise tooling across a number of different 18 | wsgi services, and improve the status quo along the way. Some of the 19 | inspiration came from `Finagle `_, 20 | Twitter's service framework for the JVM, particularly the operational 21 | standardisation it provides. 22 | 23 | In particular, we wanted to standardise how we monitor applications in 24 | production, including logging, metrics, errors and alerts. Talisker provides 25 | a single library to do this for all our apps, that our operations tooling can 26 | configure easily, and means the application doesn't need to care about it. 27 | 28 | Also we wanted to provide best practice features for applications to use. So we 29 | added support for structured logging, integrated with the python stdlib 30 | logging. This allows developers to add custom tags to their logs, as well as 31 | add operational tags. We also provide easy to use helpers for best practice 32 | usage of things like statsd clients and requests sessions, which were used 33 | inconsistently across our projects, with repeated performance problems. 34 | 35 | 36 | FAQ 37 | --- 38 | 39 | Some questions that have actually been asked, if not particularly 40 | frequently. 41 | 42 | 1. Why does talisker use a custom entry point? Wouldn't it be better to just be 43 | some WSGI middleware? 44 | 45 | There are 3 reasons for using a talisker specific entry point 46 | 47 | 1. to configure stdlib logging early, *before* any loggers are created 48 | 49 | 2. to allow for easy configuration of gunicorn for logging, statsd and 50 | other things. 51 | 52 | 3. to do things like automatically wrap your wsgi in app in some simple 53 | standard middleware, to provide request id tracing and other things. 54 | 55 | If it was just middleware, logging would get configured too late for 56 | gunicorn's logs to be affected, and you would need to add explicit middleware 57 | and config to your app and its gunicorn config. Doing it as an alternate 58 | entry point means you literally just switch out gunicorn for talisker, and 59 | you are good to go. 60 | 61 | 62 | 2. Why just gunicorn? Why not twistd, or waitress, etc? 63 | 64 | Simply because we use gunicorn currently. Integrating with other wsgi 65 | application runners is totally possible, twistd support is in the works, 66 | with uwsgi support on the road map. 67 | 68 | 69 | 3. Why is it called talisker? 70 | 71 | 'WSGI' sort of sounds like 'whisky' if you say it quick. One of my favourite 72 | whiskies is Talisker, I've even visited the distillery on the Isle of Skye. 73 | Also, Talisker is a heavily peated malt whisky, which is not to everyone's taste, 74 | which seemed to fit thematically with a WSGI runtime that is also very 75 | opinionated and probably not to everyone's taste. Also, it has 8 characters 76 | just like gunicorn, and it wasn't taken on PyPI. 77 | -------------------------------------------------------------------------------- /scripts/build_setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | import collections 25 | from setuptools.config import read_configuration 26 | 27 | config = read_configuration('setup.cfg') 28 | data = {} 29 | collections.OrderedDict() 30 | data.update(config['metadata']) 31 | data.update(config['options']) 32 | data['long_description'] = 'DESCRIPTION' 33 | 34 | long_description = open('README.rst').read().strip() 35 | sorted_data = collections.OrderedDict((k, data[k]) for k in sorted(data)) 36 | 37 | 38 | def print_line(k, v, indent=' '): 39 | if isinstance(v, list): 40 | print('{}{}=['.format(indent, k)) 41 | for i in v: 42 | print("{} '{}',".format(indent, i)) 43 | print('{}],'.format(indent)) 44 | elif isinstance(v, dict): 45 | print('{}{}=dict('.format(indent, k)) 46 | for k2 in sorted(v): 47 | print_line(k2, v[k2], indent + ' ') 48 | print('{}),'.format(indent)) 49 | elif isinstance(v, bool): 50 | print("{}{}={},".format(indent, k, v)) 51 | elif k == 'long_description': 52 | print("{}{}={},".format(indent, k, v)) 53 | else: 54 | print("{}{}='{}',".format(indent, k, v)) 55 | 56 | 57 | print("""#!/usr/bin/env python 58 | # 59 | # Copyright (c) 2015-2021 Canonical, Ltd. 60 | # 61 | # This file is part of Talisker 62 | # (see http://github.com/canonical-ols/talisker). 63 | # 64 | # Licensed to the Apache Software Foundation (ASF) under one 65 | # or more contributor license agreements. See the NOTICE file 66 | # distributed with this work for additional information 67 | # regarding copyright ownership. The ASF licenses this file 68 | # to you under the Apache License, Version 2.0 (the 69 | # "License"); you may not use this file except in compliance 70 | # with the License. You may obtain a copy of the License at 71 | # 72 | # http://www.apache.org/licenses/LICENSE-2.0 73 | # 74 | # Unless required by applicable law or agreed to in writing, 75 | # software distributed under the License is distributed on an 76 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 77 | # KIND, either express or implied. See the License for the 78 | # specific language governing permissions and limitations 79 | # under the License. 80 | # 81 | # 82 | # Note: this file is autogenerated from setup.cfg for older setuptools 83 | # 84 | try: 85 | from setuptools import setup 86 | except ImportError: 87 | from distutils.core import setup 88 | 89 | DESCRIPTION = ''' 90 | {} 91 | ''' 92 | 93 | setup(""".format(long_description)) 94 | 95 | for k, v in sorted_data.items(): 96 | print_line(k, v) 97 | 98 | print(')') 99 | -------------------------------------------------------------------------------- /scripts/limbo.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | """ 25 | This is an internal test tool to construct a requirements.txt with dependencies 26 | pinned to the miniumum supported versions, for use with tox. 27 | """ 28 | import os 29 | import sys 30 | import argparse 31 | 32 | from setuptools.config import read_configuration 33 | from pkg_resources import Requirement, parse_requirements 34 | 35 | 36 | def get_source(comes_from): 37 | index = comes_from.find(' (line ') 38 | if index != -1: 39 | comes_from = comes_from[:index] 40 | if comes_from.startswith('-r '): 41 | comes_from = comes_from[3:] 42 | return comes_from 43 | 44 | 45 | def print_file(filename, requirements): 46 | print('# ' + filename) 47 | for requirement in requirements: 48 | req = str(requirement) 49 | specs = [(r.version, r) for r in requirement.specifier] 50 | if specs: 51 | version, spec = min(specs) 52 | if version in spec: 53 | req = '{}=={}'.format(requirement.name, version) 54 | marker = getattr(requirement, 'marker', None) 55 | if marker: 56 | req += ';' + str(marker) 57 | else: 58 | req += ' # no explicit minimum' 59 | print(req) 60 | print() 61 | 62 | 63 | if __name__ == '__main__': 64 | parser = argparse.ArgumentParser() 65 | parser.add_argument("requirements", nargs="*") 66 | parser.add_argument("--extras", default=[]) 67 | 68 | args = parser.parse_args() 69 | if args.extras: 70 | args.extras = set(args.extras.split(',')) 71 | options = {} 72 | 73 | print('# AUTOGENERATED by {}\n# DO NOT EDIT\n#\n'.format(sys.argv[0])) 74 | 75 | if os.path.exists('setup.cfg'): 76 | options = read_configuration('setup.cfg').get('options', {}) 77 | 78 | install_requires = options.get('install_requires', []) 79 | extras_require = options.get('extras_require', {}) 80 | 81 | if install_requires: 82 | fname = 'setup.cfg:options.install_requires' 83 | print_file( 84 | fname, 85 | (Requirement.parse(l) for l in sorted(install_requires)), 86 | ) 87 | 88 | for extra, requires in sorted(extras_require.items()): 89 | if extra in args.extras: 90 | fname = 'setup.cfg:options.extras_require:' + extra 91 | print_file( 92 | fname, 93 | (Requirement.parse(l) for l in sorted(requires)), 94 | ) 95 | 96 | for filename in args.requirements: 97 | print_file(filename, parse_requirements(open(filename))) 98 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every 8 | little bit helps, and credit will always be given. 9 | 10 | Note: this project is copyrighted by Canonical, and the policy is that contributers must sign the CLA in order to contribute. More details here: https://www.ubuntu.com/legal/contributors 11 | 12 | You can contribute in many ways: 13 | 14 | Types of Contributions 15 | ---------------------- 16 | 17 | Report Bugs 18 | ~~~~~~~~~~~ 19 | 20 | Report bugs at https://github.com/canonical-ols/talisker/issues. 21 | 22 | If you are reporting a bug, please include: 23 | 24 | * Your operating system name and version. 25 | * Any details about your local setup that might be helpful in troubleshooting. 26 | * Detailed steps to reproduce the bug. 27 | 28 | Fix Bugs 29 | ~~~~~~~~ 30 | 31 | Look through the GitHub issues for bugs. Anything tagged with "bug" 32 | is open to whoever wants to implement it. 33 | 34 | Implement Features 35 | ~~~~~~~~~~~~~~~~~~ 36 | 37 | Look through the GitHub issues for features. Anything tagged with "feature" 38 | is open to whoever wants to implement it. 39 | 40 | Write Documentation 41 | ~~~~~~~~~~~~~~~~~~~ 42 | 43 | talisker could always use more documentation, whether as part of the 44 | official talisker docs, in docstrings, or even on the web in blog posts, 45 | articles, and such. 46 | 47 | Submit Feedback 48 | ~~~~~~~~~~~~~~~ 49 | 50 | The best way to send feedback is to file an issue at https://github.com/canonical-ols/talisker/issues. 51 | 52 | If you are proposing a feature: 53 | 54 | * Explain in detail how it would work. 55 | * Keep the scope as narrow as possible, to make it easier to implement. 56 | * Remember that this is a volunteer-driven project, and that contributions 57 | are welcome :) 58 | 59 | Get Started! 60 | ------------ 61 | 62 | Ready to contribute? Here's how to set up `talisker` for local development. 63 | 64 | 1. Fork the `talisker` repo on GitHub. 65 | 2. Clone your fork locally:: 66 | 67 | $ git clone git@github.com:your_name_here/talisker.git 68 | 69 | 3. run make with no arguments. This will create a virtualenv env in .env and install dependencies and run tests. 70 | 71 | 4. Create a branch for local development:: 72 | 73 | $ git checkout -b name-of-your-bugfix-or-feature 74 | 75 | Now you can make your changes locally. 76 | 77 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 78 | 79 | $ make # runs python3 tests and lint 80 | $ make tox # runs all python tests and lint 81 | 82 | 6. Commit your changes and push your branch to GitHub:: 83 | 84 | $ git add . 85 | $ git commit -m "Your detailed description of your changes." 86 | $ git push origin name-of-your-bugfix-or-feature 87 | 88 | 7. Submit a pull request through the GitHub website. 89 | 90 | Pull Request Guidelines 91 | ----------------------- 92 | 93 | Before you submit a pull request, check that it meets these guidelines: 94 | 95 | 1. The pull request should include tests. 96 | 2. If the pull request adds functionality, the docs should be updated. Put 97 | your new functionality into a function with a docstring, and add the 98 | feature to the list in README.rst. 99 | 3. The pull request should work for Python 2.7, 3.4 and 3.5, and for PyPy. Check 100 | https://travis-ci.org/canonical-ols/talisker/pull_requests 101 | and make sure that the tests pass for all supported Python versions. 102 | 103 | -------------------------------------------------------------------------------- /docs/requests.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | 4 | ======== 5 | Requests 6 | ======== 7 | 8 | Enhanced session 9 | ---------------- 10 | 11 | Talisker provides a way to upgrade a request.Session instance with a few extra 12 | features. 13 | 14 | Firstly, the X-Request-Id header will be added to the outgoing request headers. 15 | This can be used by other services to track the originating request id. We 16 | usually append incoming request id to one generated for that request, e.g.:: 17 | 18 | X-Request-Id: -. 19 | 20 | This allows simple searches to uncover all related sub requests for a specific 21 | request, also known as fanout. 22 | 23 | Secondly, we also collect metrics for outgoing requests. Specifically: 24 | 25 | * counter for all requests, broken down by host and view 26 | * counter for errors, broken down by host, type (http 5xx or connection error), 27 | view and error code (either POSIX error code or http status code, depending 28 | on type) 29 | * histogram for duration of http responses 30 | 31 | In statsd, they would be named like so:: 32 | 33 | .requests.count.. 34 | .requests.errors.... 35 | .requests.timeouts.. 36 | .requests.latency... 37 | 38 | Note: a view here is a human friendly name for the api/endpoint. If the 39 | upstream service returns an X-View-Name header in its response (e.g. is another 40 | talisker service), or if the user has given this call a name (see below), then 41 | this will be used. 42 | 43 | You can customise the name of this metric if you wish, with some keyword arguments:: 44 | 45 | session.post(..., metric_api_name='myapi', metric_host_name='myservice') 46 | 47 | will use these values in the resulting naming, in both prometheus and statsd.:: 48 | 49 | .requests.count.myservice.myapi... 50 | 51 | 52 | 53 | Session lifecycle 54 | ----------------- 55 | 56 | We found many of our services were not using session objects properly, often 57 | creating/destroying them per-request, thus not benefiting from the default 58 | connection pooling provided by requests. This is especially painful for latency 59 | when your upstream services are https, as nearly all ours are. But sessions are 60 | not thread-safe (see `this issue 61 | `_ for details), sadly, 62 | so a global session is risky. 63 | 64 | So, talisker helps by providing a simple way to have thread local sessions. 65 | 66 | 67 | Using a talisker session 68 | ------------------------ 69 | 70 | To get a base requests.Session thread local session with metrics and request id 71 | tracing::: 72 | 73 | session = talisker.requests.get_session() 74 | 75 | or use the wsgi environ:: 76 | 77 | session = environ['requests'] 78 | 79 | If you wish to use a custom subclass of Session rather than the default 80 | requests.Session, just pass the session class as an argument. Talisker will 81 | ensure there is one instance of this session subclass per thread.:: 82 | 83 | session = talisker.requests.get_session(MyCustomSessionClass) 84 | 85 | This works because talisker does not subclass Session to add metrics or 86 | requests id tracing. Instead, it adds a response hook to the session object for 87 | metrics, and decorates the send method to inject the header (ugh, but 88 | I couldn't find a better way). 89 | 90 | If you wish to use talisker's enhancements, but not the lifecycle management, 91 | you can do:: 92 | 93 | session = MySession() 94 | talisker.requests.configure(session) 95 | 96 | and session will now have metrics and id tracing. 97 | -------------------------------------------------------------------------------- /talisker/statsd.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | from contextlib import contextmanager 26 | from urllib.parse import urlparse, parse_qs 27 | 28 | from statsd import defaults 29 | from statsd.client import StatsClient 30 | 31 | import talisker 32 | from talisker.util import module_cache 33 | 34 | __all__ = ['get_client'] 35 | 36 | 37 | def parse_statsd_dsn(dsn): 38 | parsed = urlparse(dsn) 39 | host = parsed.hostname 40 | port = parsed.port or defaults.PORT 41 | prefix = None 42 | if parsed.path: 43 | prefix = parsed.path.strip('/').replace('/', '.') 44 | ipv6 = parsed.scheme in ('udp6', 'tcp6') 45 | size = int( 46 | parse_qs(parsed.query).get('maxudpsize', [defaults.MAXUDPSIZE])[0]) 47 | return host, port, prefix, size, ipv6 48 | 49 | 50 | @module_cache 51 | def get_client(): 52 | client = None 53 | dsn = talisker.get_config().statsd_dsn 54 | if dsn is None: 55 | client = DummyClient() 56 | else: 57 | if not dsn.startswith('udp'): 58 | raise Exception('Talisker only supports udp stastd client') 59 | client = StatsClient(*parse_statsd_dsn(dsn)) 60 | 61 | return client 62 | 63 | 64 | class DummyClient(StatsClient): 65 | """Mock client for statsd that can collect data when testing.""" 66 | _prefix = '' # force no prefix 67 | 68 | def __init__(self, collect=False): 69 | # Note: do *not* call super(), as that will create udp socket we don't 70 | # want. 71 | if collect: 72 | self.stats = MetricList() 73 | else: 74 | self.stats = None 75 | 76 | def _send(self, data): 77 | if self.stats is not None: 78 | self.stats.append(data) 79 | 80 | def pipeline(self): 81 | return self.__class__(collect=True) 82 | 83 | # pipeline methods 84 | def send(self): 85 | if self.stats: 86 | self.stats[:] = [] 87 | 88 | def __enter__(self): 89 | return self 90 | 91 | def __exit__(self, typ, value, tb): 92 | self.send() 93 | 94 | # test helper methods 95 | @contextmanager 96 | def collect(self): 97 | orig_stats = self.stats 98 | self.stats = [] 99 | yield self.stats 100 | self.stats = orig_stats 101 | 102 | 103 | class MetricList(list): 104 | """A container for searching a list of statsd metrics.""" 105 | 106 | def filter(self, name): 107 | filtered = self.__class__() 108 | for metric in self: 109 | if name in metric: 110 | filtered.append(metric) 111 | return filtered 112 | -------------------------------------------------------------------------------- /talisker/metrics.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | from contextlib import contextmanager 26 | import functools 27 | import logging 28 | import time 29 | 30 | import talisker.statsd 31 | 32 | 33 | try: 34 | import prometheus_client 35 | except ImportError: 36 | prometheus_client = False 37 | 38 | try: 39 | import statsd 40 | except ImportError: 41 | statsd = False 42 | 43 | 44 | logger = logging.getLogger(__name__) 45 | 46 | 47 | class Metric(): 48 | """Abstraction over prometheus and statsd metrics.""" 49 | 50 | def __init__(self, name, *args, **kwargs): 51 | self.name = name 52 | self.statsd_template = None 53 | 54 | if statsd: 55 | self.statsd_template = kwargs.pop('statsd', None) 56 | 57 | if prometheus_client: 58 | self.prometheus = self.get_type()(name, *args, **kwargs) 59 | else: 60 | self.prometheus = None 61 | 62 | # prometheus_client does some odd things, if we define this as a class 63 | # variable it doesn't work 64 | @property 65 | def metric_type(self): 66 | return None 67 | 68 | def get_type(self): 69 | if prometheus_client: 70 | return self.metric_type 71 | return None 72 | 73 | def get_statsd_name(self, labels): 74 | name = self.name.replace('_', '.') 75 | if self.statsd_template: 76 | try: 77 | name = self.statsd_template.format(name=name, **labels) 78 | except Exception: 79 | pass 80 | return name 81 | 82 | 83 | def protect(msg): 84 | def decorator(f): 85 | @functools.wraps(f) 86 | def wrapper(*args, **kwargs): 87 | try: 88 | f(*args, **kwargs) 89 | except Exception: 90 | logger.exception(msg) 91 | return wrapper 92 | return decorator 93 | 94 | 95 | class Histogram(Metric): 96 | 97 | @property 98 | def metric_type(self): 99 | return prometheus_client.Histogram 100 | 101 | @protect("Failed to collect histogram metric") 102 | def observe(self, amount, **labels): 103 | if self.prometheus: 104 | if labels: 105 | self.prometheus.labels(**labels).observe(amount) 106 | else: 107 | self.prometheus.observe(amount) 108 | 109 | if self.statsd_template: 110 | client = talisker.statsd.get_client() 111 | name = self.get_statsd_name(labels) 112 | client.timing(name, amount) 113 | 114 | @contextmanager 115 | def time(self): 116 | """Measure time in ms.""" 117 | t = time.time() 118 | yield 119 | d = time.time() - t 120 | self.observe(d * 1000) 121 | 122 | 123 | class Counter(Metric): 124 | 125 | @property 126 | def metric_type(self): 127 | return prometheus_client.Counter 128 | 129 | @protect("Failed to increment counter metric") 130 | def inc(self, amount=1, **labels): 131 | if self.prometheus: 132 | if labels: 133 | self.prometheus.labels(**labels).inc(amount) 134 | else: 135 | self.prometheus.inc(amount) 136 | 137 | if self.statsd_template: 138 | client = talisker.statsd.get_client() 139 | name = self.get_statsd_name(labels) 140 | client.incr(name, amount) 141 | -------------------------------------------------------------------------------- /docs/testing.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: python 2 | 3 | .. _testing: 4 | 5 | ===================== 6 | Testing with Talisker 7 | ===================== 8 | 9 | Talisker provides various tools to assist in unit testing application that use it's features. 10 | 11 | 12 | Test suite integration 13 | ---------------------- 14 | 15 | In order to run your tests in the same configuration as production, you should 16 | initialise Talisker for testing as early as possible in your test suite:: 17 | 18 | talisker.testing.configure_testing() 19 | 20 | This will set up Talisker logging with a logging.NullHandler on the root logger, 21 | as well as configure the global sentry client to point to dummy remote to 22 | capture sentry messages. 23 | 24 | Talisker uses some module globals, thread locals, and request contexts to 25 | store state. For unit tests, it is a good idea to ensure this state is 26 | cleared between tests, or else tests can not be properly isolated. To do so, 27 | ensure the following is run before every test:: 28 | 29 | talisker.testing.clear_all() 30 | 31 | 32 | Test Helpers 33 | ------------ 34 | 35 | Talisker provides a context manager for testing that will capture every log 36 | message, sentry report and statsd metric generated while it is active. It 37 | produces a test context that can be used to assert against these artefacts:: 38 | 39 | with talisker.testing.TestContext() as ctx: 40 | # code under test 41 | 42 | The context object collects events that happened while active, and presents 43 | them for inspection:: 44 | 45 | # ctx.statsd is an array of statsd metrics as strings 46 | self.assertEqual(ctx.statsd[0], 'some.metric:1.000000|ms') 47 | 48 | # ctx.sentry is an array of sentry messages sent, as the JSON dict that was 49 | # sent by the sentry client 50 | self.assertEqual(ctx.sentry[0]['message'] == 'my message') 51 | 52 | # ctx.logs is a talisker.testing.LogRecordList, which is essentially a list 53 | # of logging.LogRecords 54 | self.assertTrue(ctx.logs[0].msg == 'my msg') 55 | 56 | 57 | Asserting against log messages is not simple, expecially with extra dicts, so the context provides some helpers. 58 | 59 | For the most common cases of checking that something was logged:: 60 | 61 | # ctx.assert_log will assert that a log message exists, with a helpful 62 | # AssertionError message if it does not 63 | ctx.assert_log( 64 | name='app.logger', 65 | level='info', 66 | msg='my msg', 67 | extra={'foo': 'bar'}, 68 | ) 69 | 70 | 71 | The *context.logs* attribute also provides additional APIs: *filter()*, 72 | *exists()* and *find()*:: 73 | 74 | my_logs = ctx.logs.filter(name='app.logger') 75 | self.assertEqual(len(my_logs) == 2) 76 | self.assertTrue(my_logs.exists( 77 | level='info', 78 | msg='my msg', 79 | extra={'foo': 'bar'}, 80 | )) 81 | warning = my_logs.find(level='warning') 82 | self.assertIn('baz', warning.extra) 83 | 84 | 85 | These APIs will search logs that match the supplied keyword arguments, using 86 | the keyword to look up the attribute on the logging.LogRecord instances. 87 | A full list of such attributes can be found here: 88 | 89 | https://docs.python.org/3/library/logging.html#logrecord-attributes 90 | 91 | For all these APIs, the following applies: 92 | 93 | * The 'level' keyword can be a case insensitive string or an int (e.g. 'info' 94 | or logging.INFO), and the appropriate LogRecord attribute (levelname or 95 | levelno) will be used. 96 | 97 | * The 'msg' keyword is compared against the raw message. The 'message' keyword 98 | is compared against the interpolated msg % args. 99 | 100 | * Matching for strings is contains, not equality, i.e. *needle in haystack*, not just *needle 101 | == haystack*. 102 | 103 | * The extra dict is special cased: each supplied extra key/value is checked 104 | against the *LogRecord.extra* dict that Talisker adds. 105 | 106 | * *assert_log(...)* raises an AssertionError if the log does not exist. The error 107 | message includes how many matches each supplied term independently matched, 108 | to help narrow down issues. 109 | 110 | * *filter(...)* returns a LogRecordList of matching logs, so is chainable. 111 | 112 | * *find(...)* returns the first LogRecord found, or None. 113 | 114 | * *exists(...)* returns True if a matching LogRecord is found, else False 115 | -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import pytest 26 | 27 | try: 28 | import prometheus_client 29 | except ImportError: 30 | pytest.skip('need prometheus_client installed', allow_module_level=True) 31 | 32 | 33 | from talisker import metrics 34 | 35 | try: 36 | from prometheus_client.core import Sample 37 | 38 | def counter_name(n): 39 | return n 40 | 41 | except ImportError: 42 | # prometheus_client < 0.4 43 | from collections import namedtuple 44 | Sample = namedtuple('Sample', ['name', 'labels', 'value']) 45 | 46 | def counter_name(n): 47 | return n[:-6] 48 | 49 | 50 | @pytest.fixture() 51 | def registry(monkeypatch, tmpdir): 52 | registry = prometheus_client.CollectorRegistry() 53 | 54 | def get_metric(name, **labels): 55 | value = registry.get_sample_value(name, labels) 56 | return 0 if value is None else value 57 | 58 | registry.get_metric = get_metric 59 | return registry 60 | 61 | 62 | def test_histogram(context, registry): 63 | histogram = metrics.Histogram( 64 | name='test_histogram', 65 | documentation='test histogram', 66 | labelnames=['label'], 67 | statsd='{name}.{label}', 68 | registry=registry, 69 | ) 70 | 71 | labels = {'label': 'value'} 72 | count = registry.get_metric('test_histogram_count', **labels) 73 | sum_ = registry.get_metric('test_histogram_sum', **labels) 74 | bucket = registry.get_metric('test_histogram_bucket', le='2.5', **labels) 75 | 76 | histogram.observe(2.0, **labels) 77 | 78 | assert context.statsd[0] == 'test.histogram.value:2.000000|ms' 79 | assert registry.get_metric('test_histogram_count', **labels) - count == 1.0 80 | assert registry.get_metric('test_histogram_sum', **labels) - sum_ == 2.0 81 | after_bucket = registry.get_metric( 82 | 'test_histogram_bucket', le='2.5', **labels) 83 | assert after_bucket - bucket == 1.0 84 | 85 | 86 | def test_histogram_protected(context, registry): 87 | histogram = metrics.Histogram( 88 | name='test_histogram_protected', 89 | documentation='test histogram', 90 | labelnames=['label'], 91 | statsd='{name}.{label}', 92 | registry=registry, 93 | ) 94 | 95 | histogram.prometheus = 'THIS WILL RAISE' 96 | histogram.observe(1.0, label='label') 97 | context.assert_log(msg='Failed to collect histogram metric') 98 | 99 | 100 | def test_counter(context, registry): 101 | counter = metrics.Counter( 102 | name='test_counter', 103 | documentation='test counter', 104 | labelnames=['label'], 105 | statsd='{name}.{label}', 106 | registry=registry, 107 | ) 108 | 109 | labels = {'label': 'value'} 110 | count = registry.get_metric('test_counter', **labels) 111 | counter.inc(2, **labels) 112 | 113 | assert context.statsd[0] == 'test.counter.value:2|c' 114 | metric = registry.get_metric(counter_name('test_counter_total'), **labels) 115 | assert metric - count == 2 116 | 117 | 118 | def test_counter_protected(context, registry): 119 | counter = metrics.Counter( 120 | name='test_counter_protected', 121 | documentation='test counter', 122 | labelnames=['label'], 123 | statsd='{name}.{label}', 124 | ) 125 | 126 | counter.prometheus = 'THIS WILL RAISE' 127 | counter.inc(1, label='label') 128 | context.assert_log(msg='Failed to increment counter metric') 129 | -------------------------------------------------------------------------------- /tests/test_init.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import sys 26 | import subprocess 27 | 28 | import gevent 29 | from packaging import version 30 | import pytest 31 | import requests 32 | 33 | from talisker import testing 34 | 35 | 36 | SCRIPT = """ 37 | import logging 38 | import test2 39 | logging.getLogger('test').info('test __main__', extra={'foo': 'bar'}) 40 | """ 41 | 42 | 43 | @pytest.fixture 44 | def script(tmpdir): 45 | subdir = tmpdir.mkdir('pkg') 46 | py_script = subdir.join('test.py') 47 | py_script.write(SCRIPT) 48 | py_script2 = subdir.join('test2.py') 49 | py_script2.write('') 50 | return str(py_script) 51 | 52 | 53 | def test_run_entrypoint(script): 54 | entrypoint = 'talisker.run' 55 | output = subprocess.check_output( 56 | [entrypoint, script], 57 | stderr=subprocess.STDOUT, 58 | ) 59 | output = output.decode('utf8') 60 | assert 'test __main__' in output 61 | assert 'foo=bar' in output 62 | 63 | 64 | def test_module_entrypoint(script): 65 | entrypoint = 'python' 66 | output = subprocess.check_output( 67 | [entrypoint, '-m', 'talisker', script], 68 | stderr=subprocess.STDOUT, 69 | ) 70 | output = output.decode('utf8') 71 | assert 'test __main__' in output 72 | assert 'foo=bar' in output 73 | 74 | 75 | def test_run_sysexit(tmpdir): 76 | path = tmpdir.join('test.py') 77 | path.write('import sys; sys.exit(2)') 78 | with pytest.raises(subprocess.CalledProcessError) as e: 79 | subprocess.check_output( 80 | ['talisker.run', str(path)], 81 | stderr=subprocess.STDOUT, 82 | ) 83 | 84 | assert e.value.returncode == 2 85 | assert b'SystemExit' in e.value.output 86 | 87 | 88 | def test_gunicorn_entrypoint(): 89 | try: 90 | import gunicorn # noqa 91 | except ImportError: 92 | pytest.skip('need gunicorn installed') 93 | entrypoint = 'talisker' 94 | subprocess.check_output([entrypoint, '--help']) 95 | 96 | 97 | @pytest.mark.xfail 98 | def test_celery_entrypoint(): 99 | try: 100 | import celery # noqa 101 | except ImportError: 102 | pytest.skip('need celery installed') 103 | 104 | entrypoint = 'talisker.celery' 105 | subprocess.check_output([entrypoint, 'inspect', '--help']) 106 | 107 | 108 | @pytest.mark.skipif(sys.version_info[:2] != (3, 6), reason='python 3.6 only') 109 | @pytest.mark.timeout(80) 110 | def test_gunicorn_eventlet_entrypoint(): 111 | # this will error in python3.6 without our fix 112 | gunicorn = testing.GunicornProcess( 113 | app='tests.py36_async_tls:app', 114 | gunicorn='talisker.gunicorn.eventlet', 115 | args=['--worker-class=eventlet']) 116 | with gunicorn: 117 | r = requests.get(gunicorn.url('/')) 118 | assert r.status_code == 200 119 | 120 | 121 | @pytest.mark.skipif(sys.version_info[:2] != (3, 6), reason='python 3.6.only') 122 | @pytest.mark.skipif(version.parse(gevent.__version__) > version.parse("1.2.0"), 123 | reason="Only a problem on older gevent versions") 124 | @pytest.mark.timeout(80) 125 | def test_gunicorn_gevent_entrypoint(): 126 | # this will error in python3.6 without our fix 127 | gunicorn = testing.GunicornProcess( 128 | app='tests.py36_async_tls:app', 129 | gunicorn='talisker.gunicorn.gevent', 130 | args=['--worker-class=gevent']) 131 | with gunicorn: 132 | from pprint import pprint 133 | pprint(gunicorn.output) 134 | r = requests.get(gunicorn.url('/')) 135 | assert r.status_code == 200 136 | -------------------------------------------------------------------------------- /talisker/flask.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import logging 26 | 27 | import flask 28 | import talisker.sentry 29 | 30 | 31 | __all__ = [ 32 | 'sentry', 33 | 'register', 34 | 'TaliskerApp', 35 | ] 36 | 37 | 38 | if not talisker.sentry.enabled: 39 | def sentry(app, dsn=None, transport=None, **kwargs): 40 | pass 41 | 42 | else: 43 | from raven.contrib.flask import Sentry 44 | 45 | class FlaskSentry(Sentry): 46 | _client_set = False 47 | 48 | @property 49 | def client(self): 50 | """Return None if not yet set, so we actually create the client.""" 51 | if self._client_set: 52 | return talisker.sentry.get_client() 53 | else: 54 | return None 55 | 56 | @client.setter 57 | def client(self, client): 58 | """We let the flask extension create the sentry client.""" 59 | if client is not None: 60 | self._client_set = True 61 | talisker.sentry.set_client(client) 62 | 63 | def after_request(self, sender, response, *args, **kwargs): 64 | # override after_request to not clear context and transaction, as 65 | # we still need it 66 | return response 67 | 68 | def sentry(app, dsn=None, transport=None, **kwargs): 69 | """Enable sentry for a flask app, talisker style.""" 70 | # transport is just to support testing, not used in prod 71 | kwargs['logging'] = False 72 | kwargs['client_cls'] = talisker.sentry.TaliskerSentryClient 73 | kwargs['wrap_wsgi'] = False 74 | logging.getLogger(__name__).info( 75 | 'updating raven config from flask app') 76 | sentry = FlaskSentry(app, **kwargs) 77 | # tag sentry reports with the flask app 78 | sentry.client.tags['flask_app'] = app.name 79 | return sentry 80 | 81 | 82 | def add_view_name(response): 83 | 84 | name = flask.request.endpoint 85 | 86 | if name is not None and flask.current_app: 87 | try: 88 | if name in flask.current_app.view_functions: 89 | module = flask.current_app.view_functions[name].__module__ 90 | name = module + '.' + name 91 | except Exception: 92 | pass 93 | 94 | if name is None: 95 | # this is not a critical error, so just debug log it. 96 | logging.getLogger(__name__).debug('no flask view for {}'.format( 97 | flask.request.path 98 | )) 99 | else: 100 | response.headers['X-View-Name'] = name 101 | flask.request.environ['VIEW_NAME'] = name 102 | 103 | return response 104 | 105 | 106 | def setup(app): 107 | sentry(app) 108 | app.after_request(add_view_name) 109 | 110 | 111 | def register(app): 112 | """Register a flask app with talisker.""" 113 | # override flask default app logger set up 114 | if hasattr(app, 'logger_name'): 115 | app.config['LOGGER_HANDLER_POLICY'] = 'never' 116 | app._logger = logging.getLogger(app.logger_name) 117 | else: 118 | # we can just set the logger directly 119 | app.logger = logging.getLogger('flask.app') 120 | setup(app) 121 | 122 | 123 | class TaliskerApp(flask.Flask): 124 | 125 | def __init__(self, app, *args, **kwargs): 126 | super().__init__(app, *args, **kwargs) 127 | if hasattr(self, 'logger_name'): 128 | self.config['LOGGER_HANDLER_POLICY'] = 'never' 129 | self._logger = logging.getLogger(self.logger_name) 130 | else: 131 | self._logger = logging.getLogger('flask.app') 132 | setup(self) 133 | 134 | @property 135 | def logger(self): 136 | return self._logger 137 | -------------------------------------------------------------------------------- /tests/test_statsd.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import pytest 26 | from talisker import statsd 27 | 28 | 29 | def test_parse_statsd_dsn_host(): 30 | parsed = statsd.parse_statsd_dsn('udp://test.com') 31 | assert parsed == ('test.com', 8125, None, 512, False) 32 | 33 | 34 | def test_parse_statsd_dsn_port(): 35 | parsed = statsd.parse_statsd_dsn('udp://test.com:5000') 36 | assert parsed == ('test.com', 5000, None, 512, False) 37 | 38 | 39 | def test_parse_statsd_dsn_prefix(): 40 | parsed = statsd.parse_statsd_dsn('udp://test.com/foo.bar') 41 | assert parsed == ('test.com', 8125, 'foo.bar', 512, False) 42 | 43 | 44 | def test_parse_statsd_dsn_prefix_with_slashes(): 45 | parsed = statsd.parse_statsd_dsn('udp://test.com/foo/bar') 46 | assert parsed == ('test.com', 8125, 'foo.bar', 512, False) 47 | 48 | 49 | def test_parse_statsd_dsn_size(): 50 | parsed = statsd.parse_statsd_dsn('udp://test.com?maxudpsize=1024') 51 | assert parsed == ('test.com', 8125, None, 1024, False) 52 | 53 | 54 | def test_parse_statsd_dsn_ipv6(): 55 | parsed = statsd.parse_statsd_dsn('udp6://test.com') 56 | assert parsed == ('test.com', 8125, None, 512, True) 57 | 58 | 59 | def test_dummyclient_basic(no_network): 60 | client = statsd.DummyClient() 61 | 62 | # check the basic methods don't error or use network 63 | client.incr('a') 64 | client.decr('a') 65 | client.timing('a', 1) 66 | client.gauge('a', 1) 67 | client.set('a', 1) 68 | timer = client.timer('a') 69 | timer.start() 70 | timer.stop() 71 | 72 | 73 | def test_dummyclient_pipeline(no_network): 74 | client = statsd.DummyClient() 75 | with client.pipeline() as p: 76 | p.incr('a') 77 | p.decr('a') 78 | p.timing('a', 1) 79 | p.gauge('a', 1) 80 | p.set('a', 1) 81 | timer = p.timer('a') 82 | timer.start() 83 | timer.stop() 84 | 85 | assert p.stats[0] == 'a:1|c' 86 | assert p.stats[1] == 'a:-1|c' 87 | assert p.stats[2] == 'a:1.000000|ms' 88 | assert p.stats[3] == 'a:1|g' 89 | assert p.stats[4] == 'a:1|s' 90 | assert p.stats[5].startswith('a:') 91 | assert p.stats[5].endswith('|ms') 92 | 93 | 94 | def test_dummyclient_nested_pipeline(no_network): 95 | client = statsd.DummyClient() 96 | with client.pipeline() as p1: 97 | with p1.pipeline() as p2: 98 | assert p1.stats is not p2.stats 99 | 100 | 101 | def test_dummyclient_collect(no_network): 102 | client = statsd.DummyClient() 103 | with client.collect() as stats: 104 | client.incr('a') 105 | client.decr('a') 106 | client.timing('a', 1) 107 | client.gauge('a', 1) 108 | client.set('a', 1) 109 | timer = client.timer('a') 110 | timer.start() 111 | timer.stop() 112 | 113 | assert stats[0] == 'a:1|c' 114 | assert stats[1] == 'a:-1|c' 115 | assert stats[2] == 'a:1.000000|ms' 116 | assert stats[3] == 'a:1|g' 117 | assert stats[4] == 'a:1|s' 118 | assert stats[5].startswith('a:') 119 | assert stats[5].endswith('|ms') 120 | 121 | 122 | @pytest.mark.xfail 123 | def test_no_network(no_network): 124 | import socket 125 | socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 126 | 127 | 128 | def test_dummyclient_memory(no_network): 129 | client = statsd.DummyClient() 130 | assert client.stats is None 131 | for i in range(1000): 132 | client.incr('a') 133 | assert client.stats is None 134 | 135 | 136 | def test_metric_list(): 137 | 138 | m1 = 'foo.bar:1.0|ms' 139 | m2 = 'foo.baz:1.0|ms' 140 | m3 = 'qux:1.0|ms' 141 | metrics = statsd.MetricList([m1, m2, m3]) 142 | 143 | assert metrics.filter('foo') == [m1, m2] 144 | assert metrics.filter('foo').filter('baz') == [m2] 145 | -------------------------------------------------------------------------------- /tests/test_render.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2015-2021 Canonical, Ltd. 3 | # 4 | # This file is part of Talisker 5 | # (see http://github.com/canonical-ols/talisker). 6 | # 7 | # Licensed to the Apache Software Foundation (ASF) under one 8 | # or more contributor license agreements. See the NOTICE file 9 | # distributed with this work for additional information 10 | # regarding copyright ownership. The ASF licenses this file 11 | # to you under the Apache License, Version 2.0 (the 12 | # "License"); you may not use this file except in compliance 13 | # with the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, 18 | # software distributed under the License is distributed on an 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 20 | # KIND, either express or implied. See the License for the 21 | # specific language governing permissions and limitations 22 | # under the License. 23 | # 24 | 25 | import textwrap 26 | 27 | from talisker.render import ( 28 | Content, 29 | Link, 30 | Table, 31 | PreformattedText, 32 | ) 33 | 34 | 35 | def test_content_simple_string(): 36 | content = Content('test', tag='p', id='id', attrs={'foo': 'foo'}) 37 | assert content.html() == '

test

' 38 | assert content.text() == 'test\n\n' 39 | assert content._json() == ('id', 'test') 40 | assert Content('test', tag='h1').text() == 'test\n====\n\n' 41 | assert Content('test')._json() is None 42 | 43 | 44 | def test_content_escaped(): 45 | content = Content('te
st', tag='p', id='id
') 46 | assert content.html() == '

te<br>st

' 47 | assert content.text() == 'te
st\n\n' 48 | assert content._json() == ('id
', 'te
st') 49 | 50 | 51 | def test_content_not_escaped(): 52 | content = Content( 53 | 'test', tag='p', id='id
', escape=False) 54 | assert content.html() == '

test

' 55 | assert content.text() == 'test\n\n' 56 | assert content._json() == ('id
', 'test') 57 | 58 | 59 | def test_content_disabled(): 60 | assert Content('test', html=False).html() == '' 61 | assert Content('test', text=False).text() == '' 62 | assert Content('test', json=False)._json() is None 63 | 64 | 65 | def test_link(): 66 | assert Link('link', '/link').html() == 'link' 67 | assert Link('{}', '/link/{}', 'x').html() == 'x' 68 | assert Link('{foo}', '/link/{foo}', foo='bar').html() == ( 69 | 'bar' 70 | ) 71 | assert Link('link', '/link').text() == '/link' 72 | assert Link('link', '/link', host='http://example.com').text() == ( 73 | 'http://example.com/link' 74 | ) 75 | assert Link('link', '/link', id='link')._json() == ( 76 | 'link', {'href': '/link', 'text': 'link'} 77 | ) 78 | 79 | 80 | def test_table(): 81 | rows = [ 82 | ['a', 'b', 'c'], 83 | ['d', 'e', Link('foo', '/foo')], 84 | ] 85 | table = Table(rows, headers=['1', '2', '3'], id='table') 86 | 87 | assert table.html() == textwrap.dedent(""" 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 |
123
abc
defoo
109 | """).strip() 110 | 111 | # Add # characters to stop editors striping eol whitespace! 112 | assert table.text() == textwrap.dedent(""" 113 | 1 2 3 # 114 | ----------# 115 | a b c # 116 | d e /foo# 117 | ----------# 118 | 119 | """).replace('#', '').lstrip() 120 | 121 | id, obj = table._json() 122 | assert id == 'table' 123 | assert obj == [ 124 | {'1': 'a', '2': 'b', '3': 'c'}, 125 | {'1': 'd', '2': 'e', '3': {'href': '/foo', 'text': 'foo'}}, 126 | ] 127 | 128 | # special case in json for 2 column tables 129 | table2 = Table(list(dict(a=1, b=2).items()), id='table') 130 | assert table2._json() == ('table', {'a': 1, 'b': 2}) 131 | 132 | 133 | def test_preformatted(): 134 | text = textwrap.dedent(""" 135 | This is 136 | a preformatted 137 | multiline text fragment. 138 | """).strip() 139 | pre = PreformattedText(text, id='text') 140 | 141 | pre.text() == text 142 | pre.html() == '
' + text + '
'
143 |     pre._json() == ('text', text.split('\n'))
144 | 


--------------------------------------------------------------------------------
/tests/test_postgresql.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker
  5 | # (see http://github.com/canonical-ols/talisker).
  6 | #
  7 | # Licensed to the Apache Software Foundation (ASF) under one
  8 | # or more contributor license agreements.  See the NOTICE file
  9 | # distributed with this work for additional information
 10 | # regarding copyright ownership.  The ASF licenses this file
 11 | # to you under the Apache License, Version 2.0 (the
 12 | # "License"); you may not use this file except in compliance
 13 | # with the License.  You may obtain a copy of the License at
 14 | #
 15 | #   http://www.apache.org/licenses/LICENSE-2.0
 16 | #
 17 | # Unless required by applicable law or agreed to in writing,
 18 | # software distributed under the License is distributed on an
 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 20 | # KIND, either express or implied.  See the License for the
 21 | # specific language governing permissions and limitations
 22 | # under the License.
 23 | #
 24 | 
 25 | import pytest
 26 | from freezegun import freeze_time
 27 | 
 28 | try:
 29 |     import psycopg2  # NOQA
 30 | except ImportError:
 31 |     pytest.skip("skipping postgres only tests", allow_module_level=True)
 32 | 
 33 | # need for some fixtures
 34 | from tests import conftest  # noqa
 35 | from talisker.postgresql import (
 36 |     TaliskerConnection,
 37 |     prettify_sql,
 38 |     FILTERED,
 39 | )
 40 | import talisker.sentry
 41 | 
 42 | 
 43 | @pytest.fixture
 44 | def conn(postgresql):
 45 |     return TaliskerConnection(postgresql.dsn)
 46 | 
 47 | 
 48 | @pytest.fixture
 49 | def cursor(conn):
 50 |     return conn.cursor()
 51 | 
 52 | 
 53 | def test_connection_record_slow(conn, context, get_breadcrumbs):
 54 |     query = 'select * from table where id=%s'
 55 |     conn._threshold = 0
 56 |     conn._record('msg', query, (1,), 10000)
 57 |     records = context.logs.filter(name='talisker.slowqueries')
 58 |     assert records[0].extra['duration_ms'] == 10000.0
 59 |     assert records[0]._trailer == prettify_sql(query)
 60 | 
 61 | 
 62 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
 63 | def test_connection_record_fast(conn, context):
 64 |     query = 'select * from table'
 65 |     conn._record('msg', query, None, 0)
 66 |     context.assert_not_log(name='talisker.slowqueries')
 67 | 
 68 | 
 69 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
 70 | def test_connection_record_breadcrumb(conn, get_breadcrumbs):
 71 |     talisker.Context.new()
 72 |     query = 'select * from table'
 73 |     conn._record('msg', query, None, 1000)
 74 |     breadcrumb = get_breadcrumbs()[0]
 75 |     assert breadcrumb['message'] == 'msg'
 76 |     assert breadcrumb['category'] == 'sql'
 77 |     assert breadcrumb['data']['duration_ms'] == 1000.0
 78 |     assert breadcrumb['data']['connection'] == conn.safe_dsn
 79 |     assert 'query' in breadcrumb['data']
 80 | 
 81 | 
 82 | @freeze_time()
 83 | def test_cursor_sets_statement_timeout(cursor, get_breadcrumbs):
 84 |     talisker.Context.new()
 85 |     talisker.Context.set_relative_deadline(1000)
 86 |     cursor.execute('select %s', [1])
 87 |     crumbs = get_breadcrumbs()
 88 |     assert crumbs[0]['data']['query'] == 'SELECT %s'
 89 |     assert crumbs[0]['data']['timeout'] == 1000
 90 | 
 91 | 
 92 | def test_cursor_actually_times_out(cursor, get_breadcrumbs):
 93 |     talisker.Context.new()
 94 |     talisker.Context.set_relative_deadline(10)
 95 |     with pytest.raises(psycopg2.OperationalError) as err:
 96 |         cursor.execute('select pg_sleep(1)', [1])
 97 |     assert err.value.pgcode == '57014'
 98 |     breadcrumb = get_breadcrumbs()[0]
 99 |     assert breadcrumb['data']['timedout'] is True
100 |     assert breadcrumb['data']['pgcode'] == '57014'
101 |     assert breadcrumb['data']['pgerror'] == (
102 |         'ERROR:  canceling statement due to statement timeout\n'
103 |     )
104 | 
105 | 
106 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
107 | def test_cursor_execute_no_params(cursor, get_breadcrumbs):
108 |     talisker.Context.new()
109 |     cursor.execute('select 1')
110 |     breadcrumb = get_breadcrumbs()[0]
111 |     assert breadcrumb['data']['query'] == FILTERED
112 | 
113 | 
114 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
115 | def test_cursor_callproc_with_params(cursor, get_breadcrumbs):
116 |     talisker.Context.new()
117 |     cursor.execute(
118 |         """CREATE OR REPLACE FUNCTION test(integer) RETURNS integer
119 |                AS 'select $1'
120 |                LANGUAGE SQL;""")
121 |     cursor.callproc('test', [1])
122 |     breadcrumb = get_breadcrumbs()[1]
123 |     assert breadcrumb['data']['query'] == FILTERED
124 | 
125 | 
126 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
127 | def test_cursor_callproc_no_params(cursor, get_breadcrumbs):
128 |     talisker.Context.new()
129 |     cursor.execute(
130 |         """CREATE OR REPLACE FUNCTION test() RETURNS integer
131 |                AS 'select 1'
132 |                LANGUAGE SQL;""")
133 |     cursor.callproc('test')
134 |     breadcrumb = get_breadcrumbs()[0]
135 |     assert breadcrumb['data']['query'] == FILTERED
136 | 


--------------------------------------------------------------------------------
/tests/conftest.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker (see http://github.com/canonical-ols/talisker).
  5 | #
  6 | # Licensed to the Apache Software Foundation (ASF) under one
  7 | # or more contributor license agreements.  See the NOTICE file
  8 | # distributed with this work for additional information
  9 | # regarding copyright ownership.  The ASF licenses this file
 10 | # to you under the Apache License, Version 2.0 (the
 11 | # "License"); you may not use this file except in compliance
 12 | # with the License.  You may obtain a copy of the License at
 13 | #
 14 | #   http://www.apache.org/licenses/LICENSE-2.0
 15 | #
 16 | # Unless required by applicable law or agreed to in writing,
 17 | # software distributed under the License is distributed on an
 18 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 19 | # KIND, either express or implied.  See the License for the
 20 | # specific language governing permissions and limitations
 21 | # under the License.
 22 | #
 23 | 
 24 | import logging
 25 | import os
 26 | from wsgiref.util import setup_testing_defaults
 27 | 
 28 | import pytest
 29 | 
 30 | # do this as early as possible, to set up logging in pytest
 31 | import talisker.logs
 32 | import talisker.util
 33 | 
 34 | # set up default talisker tests with actual formatting
 35 | talisker.logs.configure_test_logging(logging.FileHandler('/dev/null'))
 36 | talisker.util.flush_early_logs()
 37 | talisker.logs.supress_noisy_logs()
 38 | 
 39 | # make sure prometheus is setup in multiprocess mode. We don't actually use
 40 | # this dir in tests, as each test gets it's own directory, but this ensures
 41 | # prometheus_client is imported in multiprocess mode
 42 | from talisker.prometheus import setup_prometheus_multiproc
 43 | setup_prometheus_multiproc(async_mode=False)
 44 | 
 45 | import talisker.context
 46 | import talisker.config
 47 | import talisker.logs
 48 | import talisker.util
 49 | import talisker.celery
 50 | import talisker.sentry
 51 | import talisker.testing
 52 | 
 53 | # set up test test sentry client
 54 | talisker.sentry.configure_testing(talisker.testing.TEST_SENTRY_DSN)
 55 | 
 56 | # create the sentry client up front
 57 | if talisker.sentry.enabled:
 58 |     talisker.sentry.get_client()
 59 | 
 60 | # clear up any initial contexts created from startup code
 61 | talisker.context.Context.clear()
 62 | 
 63 | 
 64 | @pytest.yield_fixture(autouse=True)
 65 | def clean_up(request, tmpdir, monkeypatch, config):
 66 |     """Clean up all globals.
 67 | 
 68 |     Sadly, talisker uses some global state.  Namely, stdlib logging module
 69 |     globals and thread/greenlet locals. This fixure ensures they are all
 70 |     cleaned up each time.
 71 |     """
 72 | 
 73 |     multiproc = tmpdir.mkdir('multiproc')
 74 |     monkeypatch.setenv('prometheus_multiproc_dir', str(multiproc))
 75 |     orig_client = talisker.sentry._client
 76 | 
 77 |     yield
 78 | 
 79 |     talisker.testing.clear_all()
 80 |     # some tests mess with the sentry client
 81 |     talisker.sentry.set_client(orig_client)
 82 | 
 83 |     # reset stdlib logging
 84 |     talisker.logs.reset_logging()
 85 |     talisker.logs.configure_test_logging(logging.FileHandler('/dev/null'))
 86 | 
 87 |     # reset metrics
 88 |     talisker.testing.reset_prometheus()
 89 | 
 90 | 
 91 | @pytest.fixture
 92 | def environ():
 93 |     return {
 94 |         # or else we get talisker's git hash when running tests
 95 |         'TALISKER_REVISION_ID': 'test-rev-id',
 96 |     }
 97 | 
 98 | 
 99 | @pytest.fixture
100 | def config(environ):
101 |     config = talisker.config.Config(environ)
102 |     talisker.get_config.raw_update(config)
103 |     return config
104 | 
105 | 
106 | @pytest.fixture
107 | def wsgi_env():
108 |     talisker.Context.new()
109 |     env = {'REMOTE_ADDR': '127.0.0.1'}
110 |     setup_testing_defaults(env)
111 |     return env
112 | 
113 | 
114 | @pytest.fixture
115 | def context():
116 |     ctx = talisker.testing.TestContext()
117 |     ctx.start()
118 |     yield ctx
119 |     ctx.stop()
120 | 
121 | 
122 | @pytest.fixture
123 | def django(monkeypatch):
124 |     root = os.path.dirname(__file__)
125 |     monkeypatch.setitem(
126 |         os.environ,
127 |         'PYTHONPATH',
128 |         os.path.join(root, 'django_app')
129 |     )
130 | 
131 | 
132 | @pytest.fixture
133 | def no_network(monkeypatch):
134 |     import socket
135 | 
136 |     def bad_socket(*args, **kwargs):
137 |         assert 0, "socket.socket was used!"
138 | 
139 |     monkeypatch.setattr(socket, 'socket', bad_socket)
140 | 
141 | 
142 | try:
143 |     import raven.context
144 | except ImportError:
145 |     @pytest.fixture
146 |     def get_breadcrumbs():
147 |         return lambda: None
148 | else:
149 |     @pytest.fixture
150 |     def get_breadcrumbs():
151 |         with raven.context.Context() as ctx:
152 |             yield ctx.breadcrumbs.get_buffer
153 | 
154 | 
155 | @pytest.fixture
156 | def celery_signals():
157 |     talisker.celery.enable_signals()
158 |     yield
159 |     talisker.celery.disable_signals()
160 | 
161 | 
162 | def require_module(module):
163 |     try:
164 |         __import__(module)
165 |     except ImportError:
166 |         return pytest.mark.skip(reason='{} is not installed'.format(module))
167 |     else:
168 |         return lambda f: f
169 | 


--------------------------------------------------------------------------------
/talisker/logstash/talisker.filter:
--------------------------------------------------------------------------------
  1 | filter {
  2 |   if [type] == "talisker" {
  3 |     multiline {
  4 |         pattern => "^%{TIMESTAMP_ISO8601}"
  5 |         negate => true
  6 |         what => "previous"
  7 |     }
  8 |     ruby {
  9 |       code => "
 10 |         if event['message'].length > 20000
 11 |             event.tag 'message truncated'
 12 |             event['message'] = event['message'][0..20000] + '...'
 13 |         end
 14 |       "
 15 |     }
 16 |     grok {
 17 |         patterns_dir => "./patterns"
 18 |         match => {"message"=>"(?m)%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{MODULE:module} \"%{ESCAPED_QUOTES:logmsg}(\"|(\.\.\.))( ?)%{OPTIONAL_REST_OF_LINE:logfmt}(\n?)%{GREEDYDATA:traceback}"}
 19 |         remove_field => [ "message" ]
 20 |     }
 21 |     # this filter is based on the code from the logfmt ruby gem
 22 |     ruby {
 23 |         init => "
 24 |             DOUBLEQUOTE = 34.chr
 25 |             ESCAPE = 92.chr
 26 | 
 27 |             GARBAGE = 0
 28 |             KEY = 1
 29 |             EQUAL = 2
 30 |             IVALUE = 3
 31 |             QVALUE = 4
 32 | 
 33 |             def numeric?(s)
 34 |             s.is_a?(Numeric) || s.to_s.match(/\A[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\Z/)
 35 |             end
 36 | 
 37 |             def integer?(s)
 38 |             s.is_a?(Integer) || s.to_s.match(/\A[-+]?[0-9]+\Z/)
 39 |             end
 40 |         "
 41 |         code => "
 42 |             line = event['logfmt']
 43 |             if not line
 44 |                 return [event]
 45 |             end
 46 | 
 47 |             key, value = '', ''
 48 |             escaped = false
 49 |             state = GARBAGE
 50 |             i = 0
 51 |             line.each_char do |c|
 52 |                 i += 1
 53 |                 if state == GARBAGE
 54 |                     if c > ' ' && c != DOUBLEQUOTE && c != '='
 55 |                         key = c
 56 |                         state = KEY
 57 |                     end
 58 |                     next
 59 |                 end
 60 |                 if state == KEY
 61 |                     if c > ' ' && c != DOUBLEQUOTE && c != '='
 62 |                         state = KEY
 63 |                         key << c
 64 |                     elsif c == '='
 65 |                         event[key.strip] = true
 66 |                         state = EQUAL
 67 |                     else
 68 |                         event[key.strip] = true
 69 |                         state = GARBAGE
 70 |                     end
 71 |                     event[key.strip] = true if i >= line.length
 72 |                     next
 73 |                 end
 74 |                 if state == EQUAL
 75 |                     if c > ' ' && c != DOUBLEQUOTE && c != '='
 76 |                         value = c
 77 |                         state = IVALUE
 78 |                     elsif c == DOUBLEQUOTE
 79 |                         value = ''
 80 |                         escaped = false
 81 |                         state = QVALUE
 82 |                     else
 83 |                         # modification: 'foo= ' is parsed as foo=nil, rather foo=true
 84 |                         event[key.strip] = nil
 85 |                         state = GARBAGE
 86 |                     end
 87 |                     if i >= line.length
 88 |                         if integer?(value)
 89 |                             value = value.to_i
 90 |                         elsif numeric?(value)
 91 |                             value = value.to_f
 92 |                         end
 93 |                         event[key.strip] = value || true
 94 |                     end
 95 |                     next
 96 |                 end
 97 |                 if state == IVALUE
 98 |                     if !(c > ' ' && c != DOUBLEQUOTE)
 99 |                         if integer?(value)
100 |                             value = value.to_i
101 |                         elsif numeric?(value)
102 |                             value = value.to_f
103 |                         end
104 |                         event[key.strip] = value
105 |                         state = GARBAGE
106 |                     else
107 |                         value << c
108 |                     end
109 |                     if i >= line.length
110 |                         if integer?(value)
111 |                             value = value.to_i
112 |                         elsif numeric?(value)
113 |                             value = value.to_f
114 |                         end
115 |                         event[key.strip] = value
116 |                     end
117 |                     next
118 |                 end
119 |                 if state == QVALUE
120 |                     if c == ESCAPE
121 |                         escaped = true
122 |                         value << ESCAPE
123 |                     elsif c == DOUBLEQUOTE
124 |                         if escaped
125 |                             escaped = false
126 |                             value.chop! << c
127 |                             next
128 |                         end
129 |                         event[key.strip] = value
130 |                         state = GARBAGE
131 |                     else
132 |                         escaped = false
133 |                         value << c
134 |                     end
135 |                     next
136 |                 end
137 |             end
138 |             [event]
139 |         "
140 |         remove_field => ["logfmt"]
141 |     }
142 |     date {
143 |         match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSZ"]
144 |         remove_field => [ "timestamp" ]
145 |     }
146 |   }
147 | }
148 | 


--------------------------------------------------------------------------------
/tests/django_app/django_app/settings.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker
  5 | # (see http://github.com/canonical-ols/talisker).
  6 | #
  7 | # Licensed to the Apache Software Foundation (ASF) under one
  8 | # or more contributor license agreements.  See the NOTICE file
  9 | # distributed with this work for additional information
 10 | # regarding copyright ownership.  The ASF licenses this file
 11 | # to you under the Apache License, Version 2.0 (the
 12 | # "License"); you may not use this file except in compliance
 13 | # with the License.  You may obtain a copy of the License at
 14 | #
 15 | #   http://www.apache.org/licenses/LICENSE-2.0
 16 | #
 17 | # Unless required by applicable law or agreed to in writing,
 18 | # software distributed under the License is distributed on an
 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 20 | # KIND, either express or implied.  See the License for the
 21 | # specific language governing permissions and limitations
 22 | # under the License.
 23 | #
 24 | """
 25 | Django settings for django_app project.
 26 | 
 27 | Generated by 'django-admin startproject' using Django 1.10.5.
 28 | 
 29 | For more information on this file, see
 30 | https://docs.djangoproject.com/en/1.10/topics/settings/
 31 | 
 32 | For the full list of settings and their values, see
 33 | https://docs.djangoproject.com/en/1.10/ref/settings/
 34 | """
 35 | 
 36 | import os
 37 | from talisker.postgresql import TaliskerConnection
 38 | 
 39 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 40 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 41 | 
 42 | 
 43 | # Quick-start development settings - unsuitable for production
 44 | # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
 45 | 
 46 | # SECURITY WARNING: keep the secret key used in production secret!
 47 | SECRET_KEY = 'gt#*a_v^=a2&n=b&)icz=eh@cn%je$#i18%mdqu%!k$4i0&-d8'
 48 | 
 49 | # SECURITY WARNING: don't run with debug turned on in production!
 50 | DEBUG = True
 51 | 
 52 | ALLOWED_HOSTS = ['*']
 53 | 
 54 | 
 55 | # Application definition
 56 | 
 57 | INSTALLED_APPS = [
 58 |     'django.contrib.admin',
 59 |     'django.contrib.auth',
 60 |     'django.contrib.contenttypes',
 61 |     'django.contrib.sessions',
 62 |     'django.contrib.messages',
 63 |     'django.contrib.staticfiles',
 64 |     'django_app',
 65 | ]
 66 | 
 67 | try:
 68 |     import raven  # noqa
 69 | except ImportError:
 70 |     pass
 71 | else:
 72 |     INSTALLED_APPS.insert(0, 'raven.contrib.django.raven_compat')
 73 |     SENTRY_CLIENT = 'talisker.django.SentryClient'
 74 |     SENTRY_TRANSPORT = 'talisker.sentry.DummySentryTransport'
 75 |     SENTRY_DSN = 'https://user:pass@test.com/project'
 76 | 
 77 | LOGGING_CONFIG = None
 78 | CELERY_BROKER_URL = 'redis://localhost:6379/0'
 79 | CELERY_ACCEPT_CONTENT = ['json']
 80 | CELERY_TASK_SERIALIZER = 'json'
 81 | CELERY_RESULT_SERIALIZER = 'json'
 82 | 
 83 | 
 84 | MIDDLEWARE = [
 85 |     'django.middleware.security.SecurityMiddleware',
 86 |     'django.contrib.sessions.middleware.SessionMiddleware',
 87 |     'django.middleware.common.CommonMiddleware',
 88 |     'django.middleware.csrf.CsrfViewMiddleware',
 89 |     'django.contrib.auth.middleware.AuthenticationMiddleware',
 90 |     'django.contrib.messages.middleware.MessageMiddleware',
 91 |     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 92 |     'talisker.django.middleware',
 93 | ]
 94 | 
 95 | ROOT_URLCONF = 'django_app.urls'
 96 | 
 97 | TEMPLATES = [
 98 |     {
 99 |         'BACKEND': 'django.template.backends.django.DjangoTemplates',
100 |         'DIRS': [],
101 |         'APP_DIRS': True,
102 |         'OPTIONS': {
103 |             'context_processors': [
104 |                 'django.template.context_processors.debug',
105 |                 'django.template.context_processors.request',
106 |                 'django.contrib.auth.context_processors.auth',
107 |                 'django.contrib.messages.context_processors.messages',
108 |             ],
109 |         },
110 |     },
111 | ]
112 | 
113 | WSGI_APPLICATION = 'django_app.wsgi.application'
114 | 
115 | 
116 | # Database
117 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases
118 | 
119 | DATABASES = {
120 |     'default': {
121 |         'ENGINE': 'django.db.backends.postgresql',
122 |         'HOST': 'localhost',
123 |         'NAME': 'django_app',
124 |         'USER': 'django_app',
125 |         'PASSWORD': 'django_app',
126 |         'OPTIONS': {
127 |             'connection_factory': TaliskerConnection
128 |         }
129 |     }
130 | }
131 | 
132 | 
133 | # Password validation
134 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
135 | 
136 | AUTH_PASSWORD_VALIDATORS = [
137 |     {
138 |         'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
139 |     },
140 |     {
141 |         'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
142 |     },
143 |     {
144 |         'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
145 |     },
146 |     {
147 |         'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
148 |     },
149 | ]
150 | 
151 | 
152 | # Internationalization
153 | # https://docs.djangoproject.com/en/1.10/topics/i18n/
154 | 
155 | LANGUAGE_CODE = 'en-us'
156 | 
157 | TIME_ZONE = 'UTC'
158 | 
159 | USE_I18N = True
160 | 
161 | USE_L10N = True
162 | 
163 | USE_TZ = True
164 | 
165 | 
166 | # Static files (CSS, JavaScript, Images)
167 | # https://docs.djangoproject.com/en/1.10/howto/static-files/
168 | 
169 | STATIC_URL = '/static/'
170 | STATIC_ROOT = os.path.join(os.path.normpath(os.path.dirname(__file__)), 'static')
171 | 


--------------------------------------------------------------------------------
/tests/test_celery.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker
  5 | # (see http://github.com/canonical-ols/talisker).
  6 | #
  7 | # Licensed to the Apache Software Foundation (ASF) under one
  8 | # or more contributor license agreements.  See the NOTICE file
  9 | # distributed with this work for additional information
 10 | # regarding copyright ownership.  The ASF licenses this file
 11 | # to you under the Apache License, Version 2.0 (the
 12 | # "License"); you may not use this file except in compliance
 13 | # with the License.  You may obtain a copy of the License at
 14 | #
 15 | #   http://www.apache.org/licenses/LICENSE-2.0
 16 | #
 17 | # Unless required by applicable law or agreed to in writing,
 18 | # software distributed under the License is distributed on an
 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 20 | # KIND, either express or implied.  See the License for the
 21 | # specific language governing permissions and limitations
 22 | # under the License.
 23 | #
 24 | 
 25 | import logging
 26 | import subprocess
 27 | 
 28 | import pytest
 29 | 
 30 | try:
 31 |     import celery
 32 | except ImportError:
 33 |     pytest.skip("skipping celery only tests", allow_module_level=True)
 34 | 
 35 | from celery.utils.log import get_task_logger
 36 | 
 37 | from freezegun import freeze_time
 38 | import pytest
 39 | import talisker.celery
 40 | import talisker.logs
 41 | import talisker.requests
 42 | import talisker.testing
 43 | 
 44 | 
 45 | DATESTRING = '2016-01-02 03:04:05.1234'
 46 | TIMESTAMP = 1451703845.1234
 47 | 
 48 | 
 49 | # make enqueue always take 1s
 50 | def before_task_publish(sender, body, headers, **kwargs):
 51 |     store = talisker.celery.get_store(body, headers)
 52 |     t = store.get(talisker.celery.ENQUEUE_START)
 53 |     if t:
 54 |         store[talisker.celery.ENQUEUE_START] = t - 1.0
 55 | 
 56 | 
 57 | # make task run always take 2s
 58 | def task_prerun(sender, task_id, task, **kwargs):
 59 |     task.talisker_timestamp -= 2.0
 60 | 
 61 | 
 62 | @pytest.fixture
 63 | def celery_app():
 64 |     # reregister all the signals and sentry clients
 65 |     talisker.celery.enable_signals()
 66 |     celery.signals.before_task_publish.connect(before_task_publish)
 67 |     celery.signals.task_prerun.connect(task_prerun)
 68 | 
 69 |     yield celery.Celery(broker='memory://localhost/')
 70 | 
 71 |     celery.signals.before_task_publish.disconnect(before_task_publish)
 72 |     celery.signals.task_prerun.disconnect(task_prerun)
 73 |     talisker.celery.disable_signals()
 74 | 
 75 | 
 76 | @freeze_time(DATESTRING)
 77 | def test_celery_task_enqueue(celery_app, context):
 78 |     celery_app.send_task('test_task')
 79 | 
 80 |     assert context.statsd == [
 81 |         'celery.count.test_task:1|c',
 82 |         'celery.latency.enqueue.test_task:1000.000000|ms',
 83 |     ]
 84 | 
 85 | 
 86 | @freeze_time(DATESTRING)
 87 | def test_celery_task_run(celery_app, context):
 88 |     request_id = 'myid'
 89 |     task_id = 'task_id'
 90 | 
 91 |     @celery_app.task
 92 |     def dummy_task():
 93 |         logging.getLogger(__name__).info('stdlib')
 94 |         # test celery's special task logger
 95 |         get_task_logger(__name__).info('task')
 96 | 
 97 |     dummy_task.apply(
 98 |         task_id=task_id,
 99 |         headers={
100 |             talisker.celery.REQUEST_ID: request_id,
101 |             talisker.celery.ENQUEUE_START: TIMESTAMP - 1.0
102 |         })
103 | 
104 |     assert context.statsd.filter('celery.') == [
105 |         'celery.latency.queue.tests.test_celery.dummy_task:1000.000000|ms',
106 |         'celery.success.tests.test_celery.dummy_task:1|c',
107 |         'celery.latency.run.tests.test_celery.dummy_task:2000.000000|ms',
108 |     ]
109 | 
110 |     extra = dict(
111 |         task_id=task_id,
112 |         task_name=dummy_task.name,
113 |         request_id=request_id,
114 |     )
115 |     context.assert_log(name=__name__, msg='stdlib', extra=extra)
116 |     context.assert_log(name=__name__, msg='task', extra=extra)
117 | 
118 | 
119 | @freeze_time(DATESTRING)
120 | def test_celery_task_run_retries(celery_app, context):
121 | 
122 |     @celery_app.task(bind=True)
123 |     def job_retry(self):
124 |         try:
125 |             raise Exception('failed task')
126 |         except Exception:
127 |             self.retry(countdown=1, max_retries=2)
128 | 
129 |     job_retry.apply()
130 | 
131 |     assert context.statsd.filter('celery.') == [
132 |         'celery.retry.tests.test_celery.job_retry:1|c',
133 |         'celery.latency.run.tests.test_celery.job_retry:2000.000000|ms',
134 |         'celery.retry.tests.test_celery.job_retry:1|c',
135 |         'celery.latency.run.tests.test_celery.job_retry:2000.000000|ms',
136 |         'celery.failure.tests.test_celery.job_retry:1|c',
137 |         'celery.latency.run.tests.test_celery.job_retry:2000.000000|ms',
138 |     ]
139 | 
140 | 
141 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
142 | @freeze_time(DATESTRING)
143 | def test_celery_sentry(celery_app, context):
144 |     request_id = 'myid'
145 |     task_id = 'task_id'
146 | 
147 |     @celery_app.task
148 |     def error_task():
149 |         raise Exception('error')
150 | 
151 |     error_task.apply(
152 |         task_id=task_id,
153 |         headers={talisker.celery.REQUEST_ID: request_id},
154 |     )
155 | 
156 |     assert context.statsd.filter('celery.') == [
157 |         'celery.failure.tests.test_celery.error_task:1|c',
158 |         'celery.latency.run.tests.test_celery.error_task:2000.000000|ms',
159 |     ]
160 | 
161 |     assert len(context.sentry) == 1
162 |     msg = context.sentry[0]
163 |     assert msg['extra']['task_name'] == error_task.name
164 |     assert 'task_id' in msg['extra']
165 |     assert msg['tags']['request_id'] == request_id
166 | 
167 | 
168 | @pytest.mark.xfail
169 | def test_celery_entrypoint():
170 |     entrypoint = 'talisker.celery'
171 |     subprocess.check_output([entrypoint, 'inspect', '--help'])
172 | 


--------------------------------------------------------------------------------
/tests/test_util.py:
--------------------------------------------------------------------------------
  1 | 
  2 | #
  3 | # Copyright (c) 2015-2021 Canonical, Ltd.
  4 | #
  5 | # This file is part of Talisker
  6 | # (see http://github.com/canonical-ols/talisker).
  7 | #
  8 | # Licensed to the Apache Software Foundation (ASF) under one
  9 | # or more contributor license agreements.  See the NOTICE file
 10 | # distributed with this work for additional information
 11 | # regarding copyright ownership.  The ASF licenses this file
 12 | # to you under the Apache License, Version 2.0 (the
 13 | # "License"); you may not use this file except in compliance
 14 | # with the License.  You may obtain a copy of the License at
 15 | #
 16 | #   http://www.apache.org/licenses/LICENSE-2.0
 17 | #
 18 | # Unless required by applicable law or agreed to in writing,
 19 | # software distributed under the License is distributed on an
 20 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 21 | # KIND, either express or implied.  See the License for the
 22 | # specific language governing permissions and limitations
 23 | # under the License.
 24 | #
 25 | 
 26 | import os
 27 | import threading
 28 | import time
 29 | 
 30 | from freezegun import freeze_time
 31 | from talisker import util
 32 | 
 33 | 
 34 | def test_sanitize_url():
 35 |     source = 'https://user:pass@host/path?q=bar'
 36 |     expected = 'https://user:********@host/path?'
 37 |     assert util.sanitize_url(source) == expected
 38 | 
 39 |     # with port
 40 |     assert (
 41 |         util.sanitize_url('https://host:1234/path') == 'https://host:1234/path'
 42 |     )
 43 | 
 44 |     # no user
 45 |     assert util.sanitize_url('https://host/path') == 'https://host/path'
 46 | 
 47 | 
 48 | @freeze_time('2016-01-02 03:04:05.1234')
 49 | def test_get_rounded_ms():
 50 |     assert util.get_rounded_ms(time.time() - 1.0) == 1000
 51 |     assert util.get_rounded_ms(time.time() - 123.0) == 123000
 52 |     assert util.get_rounded_ms(time.time() - 0.123) == 123
 53 |     assert util.get_rounded_ms(time.time() - 0.123456789) == 123.457
 54 |     assert util.get_rounded_ms(0.1, 0.3) == 200.0
 55 | 
 56 | 
 57 | def test_get_root_exception_implicit():
 58 |     exc = None
 59 |     try:
 60 |         try:
 61 |             try:
 62 |                 raise Exception('root')
 63 |             except Exception:
 64 |                 raise Exception('one')
 65 |         except Exception:
 66 |             raise Exception('two')
 67 |     except Exception as e:
 68 |         exc = e
 69 | 
 70 |     root = util.get_root_exception(exc)
 71 |     assert root.args == ('root',)
 72 | 
 73 | 
 74 | def test_get_root_exception_explicit():
 75 |     exc = None
 76 |     try:
 77 |         try:
 78 |             try:
 79 |                 raise Exception('root')
 80 |             except Exception as a:
 81 |                 raise Exception('one') from a
 82 |         except Exception as b:
 83 |             raise Exception('two') from b
 84 |     except Exception as c:
 85 |         exc = c
 86 |     root = util.get_root_exception(exc)
 87 |     assert root.args == ('root',)
 88 | 
 89 | 
 90 | def test_get_root_exception_mixed():
 91 |     exc = None
 92 |     try:
 93 |         try:
 94 |             try:
 95 |                 raise Exception('root')
 96 |             except Exception as a:
 97 |                 raise Exception('one') from a
 98 |         except Exception:
 99 |             raise Exception('two')
100 |     except Exception as e:
101 |         exc = e
102 |     root = util.get_root_exception(exc)
103 |     assert root.args == ('root',)
104 | 
105 | 
106 | def test_get_errno_fields_permissions():
107 |     exc = None
108 |     try:
109 |         open('/blah', 'w')
110 |     except Exception as e:
111 |         exc = e
112 | 
113 |     assert util.get_errno_fields(exc) == {
114 |         'errno': 'EACCES',
115 |         'strerror': 'Permission denied',
116 |         'filename': '/blah',
117 |     }
118 | 
119 | 
120 | def test_get_errno_fields_connection():
121 |     exc = None
122 |     try:
123 |         import socket
124 |         s = socket.socket()
125 |         s.connect(('localhost', 54321))
126 |     except Exception as e:
127 |         exc = e
128 | 
129 |     assert util.get_errno_fields(exc) == {
130 |         'errno': 'ECONNREFUSED',
131 |         'strerror': 'Connection refused',
132 |     }
133 | 
134 | 
135 | def test_get_errno_fields_dns():
136 |     exc = None
137 |     try:
138 |         import socket
139 |         import platform
140 |         s = socket.socket()
141 |         s.connect(('some-host-name-that-will-not-resolve.com', 54321))
142 |     except Exception as e:
143 |         exc = e
144 | 
145 |     processed_exc = util.get_errno_fields(exc)
146 |     if platform.system() == 'Darwin':
147 |         assert processed_exc == {
148 |             'errno': 'ENOEXEC',
149 |             'strerror': 'nodename nor servname provided, or not known'
150 |         }
151 |     else:
152 |         assert processed_exc in [{
153 |             'errno': 'EAI_NONAME',
154 |             'strerror': 'Name or service not known'
155 |         }, {
156 |             'errno': 'EAI_NODATA',
157 |             'strerror': 'No address associated with hostname'
158 |         }]
159 | 
160 | 
161 | def test_local_forking():
162 |     local = util.Local()
163 |     local.test = 1
164 | 
165 |     if os.fork() == 0:
166 |         assert not hasattr(local, 'test')
167 |         local.test = 2
168 |         assert local.test == 2
169 |         os._exit(0)
170 | 
171 | 
172 | def test_local_threading():
173 |     local = util.Local()
174 |     local.test = 1
175 | 
176 |     thread_results = {}
177 | 
178 |     def f(results):
179 |         results['no_attr'] = not hasattr(local, 'test')
180 |         try:
181 |             local.test = 2
182 |         except Exception:
183 |             pass
184 |         else:
185 |             results['new_attr'] = local.test
186 | 
187 |     thread = threading.Thread(target=f, args=(thread_results,))
188 |     thread.start()
189 |     thread.join(timeout=1.1)
190 | 
191 |     assert local.test == 1
192 |     assert thread_results['no_attr']
193 |     assert thread_results['new_attr'] == 2
194 | 


--------------------------------------------------------------------------------
/tests/test_flask.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker
  5 | # (see http://github.com/canonical-ols/talisker).
  6 | #
  7 | # Licensed to the Apache Software Foundation (ASF) under one
  8 | # or more contributor license agreements.  See the NOTICE file
  9 | # distributed with this work for additional information
 10 | # regarding copyright ownership.  The ASF licenses this file
 11 | # to you under the Apache License, Version 2.0 (the
 12 | # "License"); you may not use this file except in compliance
 13 | # with the License.  You may obtain a copy of the License at
 14 | #
 15 | #   http://www.apache.org/licenses/LICENSE-2.0
 16 | #
 17 | # Unless required by applicable law or agreed to in writing,
 18 | # software distributed under the License is distributed on an
 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 20 | # KIND, either express or implied.  See the License for the
 21 | # specific language governing permissions and limitations
 22 | # under the License.
 23 | #
 24 | 
 25 | import logging
 26 | 
 27 | import pytest
 28 | 
 29 | try:
 30 |     from flask import Flask
 31 | except ImportError:
 32 |     pytest.skip('skipping flask tests', allow_module_level=True)
 33 | 
 34 | import talisker.flask
 35 | 
 36 | from talisker.testing import (
 37 |     TEST_SENTRY_DSN,
 38 |     get_sentry_messages,
 39 | )
 40 | 
 41 | 
 42 | class IgnoredException(Exception):
 43 |     pass
 44 | 
 45 | 
 46 | @pytest.fixture
 47 | def flask_app():
 48 |     app = Flask(__name__)
 49 | 
 50 |     if talisker.sentry.enabled:
 51 |         from talisker.sentry import DummySentryTransport
 52 |         app.config['SENTRY_TRANSPORT'] = DummySentryTransport
 53 |         app.config['SENTRY_DSN'] = TEST_SENTRY_DSN
 54 | 
 55 |     @app.route('/')
 56 |     def error():
 57 |         raise Exception('app exception')
 58 | 
 59 |     @app.route('/ignored')
 60 |     def ignored():
 61 |         raise IgnoredException('test exception')
 62 | 
 63 |     return app
 64 | 
 65 | 
 66 | def get_url(the_app, *args, **kwargs):
 67 |     app_client = the_app.test_client()
 68 |     with the_app.app_context():
 69 |         return app_client.get(*args, **kwargs)
 70 | 
 71 | 
 72 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
 73 | def test_flask_sentry_sends_message(flask_app):
 74 |     sentry = talisker.flask.sentry(flask_app)
 75 |     response = get_url(flask_app, '/')
 76 | 
 77 |     assert response.status_code == 500
 78 |     msgs = get_sentry_messages(sentry.client)
 79 |     assert len(msgs) == 1
 80 |     msg = msgs[0]
 81 |     if 'culprit' in msg:
 82 |         assert msg['culprit'] == '/'
 83 |     else:
 84 |         assert msg['transaction'] == '/'
 85 | 
 86 | 
 87 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
 88 | def test_flask_sentry_default_include_paths(flask_app):
 89 |     sentry = talisker.flask.sentry(flask_app)
 90 |     assert sentry.client.include_paths == set(['tests.test_flask'])
 91 | 
 92 | 
 93 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
 94 | def test_flask_sentry_app_config_ignore_exc(flask_app, monkeypatch, context):
 95 |     monkeypatch.setitem(flask_app.config, 'SENTRY_CONFIG', {
 96 |         'ignore_exceptions': ['IgnoredException']
 97 |     })
 98 |     sentry = talisker.flask.sentry(flask_app)
 99 | 
100 |     assert 'IgnoredException' in sentry.client.ignore_exceptions
101 | 
102 |     response = get_url(flask_app, '/ignored')
103 | 
104 |     assert response.status_code == 500
105 |     assert len(get_sentry_messages(sentry.client)) == 0
106 | 
107 | 
108 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
109 | def test_flask_sentry_uses_app_config_to_set_name(flask_app, monkeypatch):
110 |     monkeypatch.setitem(flask_app.config, 'SENTRY_NAME', 'SomeName')
111 |     sentry = talisker.flask.sentry(flask_app)
112 |     assert sentry.client.name == 'SomeName'
113 | 
114 |     response = get_url(flask_app, '/')
115 | 
116 |     assert response.status_code == 500
117 |     msgs = get_sentry_messages(sentry.client)
118 |     assert len(msgs) == 1
119 |     msg = msgs[0]
120 |     assert msg['server_name'] == 'SomeName'
121 | 
122 | 
123 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
124 | def test_flask_sentry_app_tag(flask_app):
125 |     sentry = talisker.flask.sentry(flask_app)
126 |     response = get_url(flask_app, '/')
127 | 
128 |     assert response.status_code == 500
129 |     msgs = get_sentry_messages(sentry.client)
130 |     assert msgs[0]['tags']['flask_app'] == flask_app.name
131 | 
132 | 
133 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
134 | def test_flask_sentry_not_clear_afer_request(monkeypatch):
135 |     app = Flask(__name__)
136 | 
137 |     @app.route('/')
138 |     def index():
139 |         return 'ok'
140 | 
141 |     sentry = talisker.flask.sentry(app)
142 |     calls = []
143 |     monkeypatch.setattr(sentry.client.context, 'clear',
144 |                         lambda: calls.append(1))
145 |     get_url(app, '/')
146 |     assert len(calls) == 0
147 |     assert isinstance(sentry, talisker.flask.FlaskSentry)
148 | 
149 | 
150 | def test_talisker_flask_app():
151 |     app = talisker.flask.TaliskerApp(__name__)
152 |     logname = getattr(app, 'logger_name', 'flask.app')
153 | 
154 |     if talisker.sentry.enabled:
155 |         assert 'sentry' in app.extensions
156 |     assert app.logger is logging.getLogger(logname)
157 | 
158 |     if 'LOGGER_HANDLER_POLICY' in app.config:
159 |         assert app.config['LOGGER_HANDLER_POLICY'] == 'never'
160 | 
161 | 
162 | def test_register_app():
163 |     app = Flask(__name__)
164 |     talisker.flask.register(app)
165 | 
166 |     logname = getattr(app, 'logger_name', 'flask.app')
167 |     if talisker.sentry.enabled:
168 |         assert 'sentry' in app.extensions
169 |     assert app.logger is logging.getLogger(logname)
170 |     if 'LOGGER_HANDLER_POLICY' in app.config:
171 |         assert app.config['LOGGER_HANDLER_POLICY'] == 'never'
172 | 
173 | 
174 | def test_flask_view_name_header():
175 |     app = Flask(__name__)
176 | 
177 |     @app.route('/')
178 |     def index():
179 |         return 'ok'
180 | 
181 |     talisker.flask.register(app)
182 |     response = get_url(app, '/')
183 |     assert response.headers['X-View-Name'] == 'tests.test_flask.index'
184 | 
185 | 
186 | def test_flask_view_name_header_no_view(context):
187 |     app = Flask(__name__)
188 | 
189 |     @app.route('/')
190 |     def index():
191 |         return 'ok'
192 | 
193 |     talisker.flask.register(app)
194 |     context.logs[:] = []
195 |     response = get_url(app, '/notexist')
196 |     assert 'X-View-Name' not in response.headers
197 |     context.assert_log(msg="no flask view for /notexist")
198 | 
199 | 
200 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='need raven installed')
201 | def test_flask_extension_updates_sentry_client():
202 |     orig_client = talisker.sentry.get_client()
203 |     app = Flask(__name__)
204 |     ext = talisker.flask.sentry(app)
205 |     assert ext.client is not orig_client
206 |     assert talisker.sentry.get_client() is not orig_client
207 | 


--------------------------------------------------------------------------------
/talisker/postgresql.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker
  5 | # (see http://github.com/canonical-ols/talisker).
  6 | #
  7 | # Licensed to the Apache Software Foundation (ASF) under one
  8 | # or more contributor license agreements.  See the NOTICE file
  9 | # distributed with this work for additional information
 10 | # regarding copyright ownership.  The ASF licenses this file
 11 | # to you under the Apache License, Version 2.0 (the
 12 | # "License"); you may not use this file except in compliance
 13 | # with the License.  You may obtain a copy of the License at
 14 | #
 15 | #   http://www.apache.org/licenses/LICENSE-2.0
 16 | #
 17 | # Unless required by applicable law or agreed to in writing,
 18 | # software distributed under the License is distributed on an
 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 20 | # KIND, either express or implied.  See the License for the
 21 | # specific language governing permissions and limitations
 22 | # under the License.
 23 | #
 24 | 
 25 | import collections
 26 | import logging
 27 | import time
 28 | 
 29 | import psycopg2
 30 | from psycopg2.extensions import cursor, connection
 31 | 
 32 | try:
 33 |     from sqlparse import format as format_sql
 34 | except ImportError:
 35 |     def format_sql(sql, *args, **kwargs):
 36 |         return sql
 37 | 
 38 | import talisker
 39 | from talisker.util import get_rounded_ms
 40 | import talisker.sentry
 41 | 
 42 | __all__ = [
 43 |     'TaliskerConnection',
 44 |     'TaliskerCursor',
 45 |     'prettify_sql',
 46 | ]
 47 | 
 48 | 
 49 | FILTERED = ''
 50 | 
 51 | 
 52 | def prettify_sql(sql):
 53 |     if sql is None:
 54 |         return None
 55 |     return format_sql(
 56 |         sql,
 57 |         keyword_case="upper",
 58 |         identfier_case="lower",
 59 |         strip_comments=False,
 60 |         reindent=True,
 61 |         indent_tabs=False)
 62 | 
 63 | 
 64 | class TaliskerConnection(connection):
 65 |     _logger = None
 66 |     _threshold = None
 67 |     _explain = None
 68 |     _safe_dsn = None
 69 |     _safe_dsn_format = '{user}@{host}:{port}/{dbname}'
 70 | 
 71 |     @property
 72 |     def safe_dsn(self):
 73 |         if self._safe_dsn is None:
 74 |             try:
 75 |                 params = self.get_dsn_parameters()
 76 |                 params.setdefault('host', 'localhost')
 77 |                 self._safe_dsn = self._safe_dsn_format.format(**params)
 78 |             except Exception:
 79 |                 self.logger.exception('Failed to parse DSN')
 80 |                 self._safe_dsn = 'could not parse dsn'
 81 | 
 82 |         return self._safe_dsn
 83 | 
 84 |     @property
 85 |     def logger(self):
 86 |         if self._logger is None:
 87 |             self._logger = logging.getLogger('talisker.slowqueries')
 88 |         return self._logger
 89 | 
 90 |     @property
 91 |     def query_threshold(self):
 92 |         if self._threshold is None:
 93 |             self._threshold = talisker.get_config().slowquery_threshold
 94 |         return self._threshold
 95 | 
 96 |     @property
 97 |     def explain_breadcrumbs(self):
 98 |         if self._explain is None:
 99 |             self._explain = talisker.get_config().explain_sql
100 |         return self._explain
101 | 
102 |     def cursor(self, *args, **kwargs):
103 |         kwargs.setdefault('cursor_factory', TaliskerCursor)
104 |         return super().cursor(*args, **kwargs)
105 | 
106 |     def _format_query(self, query, vars):
107 |         if callable(query):
108 |             query = query()
109 |         query = prettify_sql(query)
110 |         if query is None or vars is None:
111 |             return FILTERED
112 |         return query
113 | 
114 |     def _record(self, msg, query, vars, duration, extra={}):
115 |         talisker.Context.track('sql', duration)
116 | 
117 |         qdata = collections.OrderedDict()
118 |         qdata['duration_ms'] = duration
119 |         qdata['connection'] = self.safe_dsn
120 |         qdata.update(extra)
121 | 
122 |         # grab a reference here, where super() works
123 |         base_connection = super()
124 | 
125 |         if self.query_threshold >= 0 and duration > self.query_threshold:
126 |             formatted = self._format_query(query, vars)
127 |             self.logger.info(
128 |                 'slow ' + msg, extra=dict(qdata, trailer=formatted))
129 | 
130 |         def processor(data):
131 |             qdata['query'] = self._format_query(query, vars)
132 |             if self.explain_breadcrumbs or talisker.Context.debug:
133 |                 try:
134 |                     cursor = base_connection.cursor()
135 |                     cursor.execute('EXPLAIN ' + query, vars)
136 |                     plan = '\n'.join(l[0] for l in cursor.fetchall())
137 |                     qdata['plan'] = plan
138 |                 except Exception as e:
139 |                     qdata['plan'] = 'could not explain query: ' + str(e)
140 | 
141 |             data['data'].update(qdata)
142 | 
143 |         breadcrumb = dict(
144 |             message=msg, category='sql', data={}, processor=processor)
145 | 
146 |         talisker.sentry.record_breadcrumb(**breadcrumb)
147 | 
148 | 
149 | class TaliskerCursor(cursor):
150 | 
151 |     def apply_timeout(self):
152 |         ctx_timeout = talisker.Context.deadline_timeout()
153 |         if ctx_timeout is None:
154 |             return None
155 | 
156 |         ms = int(ctx_timeout * 1000)
157 |         super().execute(
158 |             'SET LOCAL statement_timeout TO %s', (ms,)
159 |         )
160 |         return ms
161 | 
162 |     def execute(self, query, vars=None):
163 |         extra = collections.OrderedDict()
164 |         timeout = self.apply_timeout()
165 |         if timeout is not None:
166 |             extra['timeout'] = timeout
167 |         timestamp = time.time()
168 |         try:
169 |             return super().execute(query, vars)
170 |         except psycopg2.OperationalError as exc:
171 |             extra['pgcode'] = exc.pgcode
172 |             extra['pgerror'] = exc.pgerror
173 |             if exc.pgcode == '57014':
174 |                 extra['timedout'] = True
175 |             raise
176 |         finally:
177 |             duration = get_rounded_ms(timestamp)
178 |             self.connection._record('query', query, vars, duration, extra)
179 | 
180 |     def callproc(self, procname, vars=None):
181 |         extra = collections.OrderedDict()
182 |         timeout = self.apply_timeout()
183 |         if timeout is not None:
184 |             extra['timeout'] = timeout
185 |         timestamp = time.time()
186 |         try:
187 |             return super().callproc(procname, vars)
188 |         except psycopg2.OperationalError as exc:
189 |             extra['pgcode'] = exc.pgcode
190 |             extra['pgerror'] = exc.pgerror
191 |             if exc.pgcode == '57014':
192 |                 extra['timedout'] = True
193 |             raise
194 |         finally:
195 |             duration = get_rounded_ms(timestamp)
196 |             # no query parameters, cannot safely record
197 |             self.connection._record(
198 |                 'stored proc: {}'.format(procname),
199 |                 None,
200 |                 vars,
201 |                 duration,
202 |                 extra,
203 |             )
204 | 


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
  1 | # Makefile for Sphinx documentation
  2 | #
  3 | 
  4 | # You can set these variables from the command line.
  5 | SPHINXOPTS    =
  6 | SPHINXBUILD   ?= sphinx-build
  7 | PAPER         =
  8 | BUILDDIR      = _build
  9 | 
 10 | # User-friendly check for sphinx-build
 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
 13 | endif
 14 | 
 15 | # Internal variables.
 16 | PAPEROPT_a4     = -D latex_paper_size=a4
 17 | PAPEROPT_letter = -D latex_paper_size=letter
 18 | ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 19 | # the i18n builder cannot share the environment and doctrees with the others
 20 | I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 21 | 
 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 23 | 
 24 | help:
 25 | 	@echo "Please use \`make ' where  is one of"
 26 | 	@echo "  html       to make standalone HTML files"
 27 | 	@echo "  dirhtml    to make HTML files named index.html in directories"
 28 | 	@echo "  singlehtml to make a single large HTML file"
 29 | 	@echo "  pickle     to make pickle files"
 30 | 	@echo "  json       to make JSON files"
 31 | 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 32 | 	@echo "  qthelp     to make HTML files and a qthelp project"
 33 | 	@echo "  devhelp    to make HTML files and a Devhelp project"
 34 | 	@echo "  epub       to make an epub"
 35 | 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 36 | 	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 37 | 	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
 38 | 	@echo "  text       to make text files"
 39 | 	@echo "  man        to make manual pages"
 40 | 	@echo "  texinfo    to make Texinfo files"
 41 | 	@echo "  info       to make Texinfo files and run them through makeinfo"
 42 | 	@echo "  gettext    to make PO message catalogs"
 43 | 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 44 | 	@echo "  xml        to make Docutils-native XML files"
 45 | 	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
 46 | 	@echo "  linkcheck  to check all external links for integrity"
 47 | 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 48 | 
 49 | clean:
 50 | 	rm -rf $(BUILDDIR)/*
 51 | 
 52 | html:
 53 | 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 54 | 	@echo
 55 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 56 | 
 57 | dirhtml:
 58 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 59 | 	@echo
 60 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 61 | 
 62 | singlehtml:
 63 | 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 64 | 	@echo
 65 | 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 66 | 
 67 | pickle:
 68 | 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 69 | 	@echo
 70 | 	@echo "Build finished; now you can process the pickle files."
 71 | 
 72 | json:
 73 | 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 74 | 	@echo
 75 | 	@echo "Build finished; now you can process the JSON files."
 76 | 
 77 | htmlhelp:
 78 | 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 79 | 	@echo
 80 | 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 81 | 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 82 | 
 83 | qthelp:
 84 | 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 85 | 	@echo
 86 | 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 87 | 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
 88 | 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/talisker.qhcp"
 89 | 	@echo "To view the help file:"
 90 | 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/talisker.qhc"
 91 | 
 92 | devhelp:
 93 | 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 94 | 	@echo
 95 | 	@echo "Build finished."
 96 | 	@echo "To view the help file:"
 97 | 	@echo "# mkdir -p $$HOME/.local/share/devhelp/talisker"
 98 | 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/talisker"
 99 | 	@echo "# devhelp"
100 | 
101 | epub:
102 | 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
103 | 	@echo
104 | 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
105 | 
106 | latex:
107 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
108 | 	@echo
109 | 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
110 | 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
111 | 	      "(use \`make latexpdf' here to do that automatically)."
112 | 
113 | latexpdf:
114 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
115 | 	@echo "Running LaTeX files through pdflatex..."
116 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
117 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
118 | 
119 | latexpdfja:
120 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
121 | 	@echo "Running LaTeX files through platex and dvipdfmx..."
122 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
123 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
124 | 
125 | text:
126 | 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
127 | 	@echo
128 | 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
129 | 
130 | man:
131 | 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
132 | 	@echo
133 | 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
134 | 
135 | texinfo:
136 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
137 | 	@echo
138 | 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
139 | 	@echo "Run \`make' in that directory to run these through makeinfo" \
140 | 	      "(use \`make info' here to do that automatically)."
141 | 
142 | info:
143 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
144 | 	@echo "Running Texinfo files through makeinfo..."
145 | 	make -C $(BUILDDIR)/texinfo info
146 | 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
147 | 
148 | gettext:
149 | 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
150 | 	@echo
151 | 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
152 | 
153 | changes:
154 | 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
155 | 	@echo
156 | 	@echo "The overview file is in $(BUILDDIR)/changes."
157 | 
158 | linkcheck:
159 | 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
160 | 	@echo
161 | 	@echo "Link check complete; look for any errors in the above output " \
162 | 	      "or in $(BUILDDIR)/linkcheck/output.txt."
163 | 
164 | doctest:
165 | 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
166 | 	@echo "Testing of doctests in the sources finished, look at the " \
167 | 	      "results in $(BUILDDIR)/doctest/output.txt."
168 | 
169 | xml:
170 | 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
171 | 	@echo
172 | 	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
173 | 
174 | pseudoxml:
175 | 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
176 | 	@echo
177 | 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
178 | 


--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
  1 | @ECHO OFF
  2 | 
  3 | REM Command file for Sphinx documentation
  4 | 
  5 | if "%SPHINXBUILD%" == "" (
  6 | 	set SPHINXBUILD=sphinx-build
  7 | )
  8 | set BUILDDIR=_build
  9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
 10 | set I18NSPHINXOPTS=%SPHINXOPTS% .
 11 | if NOT "%PAPER%" == "" (
 12 | 	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
 13 | 	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
 14 | )
 15 | 
 16 | if "%1" == "" goto help
 17 | 
 18 | if "%1" == "help" (
 19 | 	:help
 20 | 	echo.Please use `make ^` where ^ is one of
 21 | 	echo.  html       to make standalone HTML files
 22 | 	echo.  dirhtml    to make HTML files named index.html in directories
 23 | 	echo.  singlehtml to make a single large HTML file
 24 | 	echo.  pickle     to make pickle files
 25 | 	echo.  json       to make JSON files
 26 | 	echo.  htmlhelp   to make HTML files and a HTML help project
 27 | 	echo.  qthelp     to make HTML files and a qthelp project
 28 | 	echo.  devhelp    to make HTML files and a Devhelp project
 29 | 	echo.  epub       to make an epub
 30 | 	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
 31 | 	echo.  text       to make text files
 32 | 	echo.  man        to make manual pages
 33 | 	echo.  texinfo    to make Texinfo files
 34 | 	echo.  gettext    to make PO message catalogs
 35 | 	echo.  changes    to make an overview over all changed/added/deprecated items
 36 | 	echo.  xml        to make Docutils-native XML files
 37 | 	echo.  pseudoxml  to make pseudoxml-XML files for display purposes
 38 | 	echo.  linkcheck  to check all external links for integrity
 39 | 	echo.  doctest    to run all doctests embedded in the documentation if enabled
 40 | 	goto end
 41 | )
 42 | 
 43 | if "%1" == "clean" (
 44 | 	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
 45 | 	del /q /s %BUILDDIR%\*
 46 | 	goto end
 47 | )
 48 | 
 49 | 
 50 | %SPHINXBUILD% 2> nul
 51 | if errorlevel 9009 (
 52 | 	echo.
 53 | 	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
 54 | 	echo.installed, then set the SPHINXBUILD environment variable to point
 55 | 	echo.to the full path of the 'sphinx-build' executable. Alternatively you
 56 | 	echo.may add the Sphinx directory to PATH.
 57 | 	echo.
 58 | 	echo.If you don't have Sphinx installed, grab it from
 59 | 	echo.http://sphinx-doc.org/
 60 | 	exit /b 1
 61 | )
 62 | 
 63 | if "%1" == "html" (
 64 | 	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
 65 | 	if errorlevel 1 exit /b 1
 66 | 	echo.
 67 | 	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
 68 | 	goto end
 69 | )
 70 | 
 71 | if "%1" == "dirhtml" (
 72 | 	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
 73 | 	if errorlevel 1 exit /b 1
 74 | 	echo.
 75 | 	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
 76 | 	goto end
 77 | )
 78 | 
 79 | if "%1" == "singlehtml" (
 80 | 	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
 81 | 	if errorlevel 1 exit /b 1
 82 | 	echo.
 83 | 	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
 84 | 	goto end
 85 | )
 86 | 
 87 | if "%1" == "pickle" (
 88 | 	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
 89 | 	if errorlevel 1 exit /b 1
 90 | 	echo.
 91 | 	echo.Build finished; now you can process the pickle files.
 92 | 	goto end
 93 | )
 94 | 
 95 | if "%1" == "json" (
 96 | 	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
 97 | 	if errorlevel 1 exit /b 1
 98 | 	echo.
 99 | 	echo.Build finished; now you can process the JSON files.
100 | 	goto end
101 | )
102 | 
103 | if "%1" == "htmlhelp" (
104 | 	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
105 | 	if errorlevel 1 exit /b 1
106 | 	echo.
107 | 	echo.Build finished; now you can run HTML Help Workshop with the ^
108 | .hhp project file in %BUILDDIR%/htmlhelp.
109 | 	goto end
110 | )
111 | 
112 | if "%1" == "qthelp" (
113 | 	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
114 | 	if errorlevel 1 exit /b 1
115 | 	echo.
116 | 	echo.Build finished; now you can run "qcollectiongenerator" with the ^
117 | .qhcp project file in %BUILDDIR%/qthelp, like this:
118 | 	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\talisker.qhcp
119 | 	echo.To view the help file:
120 | 	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\talisker.ghc
121 | 	goto end
122 | )
123 | 
124 | if "%1" == "devhelp" (
125 | 	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
126 | 	if errorlevel 1 exit /b 1
127 | 	echo.
128 | 	echo.Build finished.
129 | 	goto end
130 | )
131 | 
132 | if "%1" == "epub" (
133 | 	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
134 | 	if errorlevel 1 exit /b 1
135 | 	echo.
136 | 	echo.Build finished. The epub file is in %BUILDDIR%/epub.
137 | 	goto end
138 | )
139 | 
140 | if "%1" == "latex" (
141 | 	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
142 | 	if errorlevel 1 exit /b 1
143 | 	echo.
144 | 	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
145 | 	goto end
146 | )
147 | 
148 | if "%1" == "latexpdf" (
149 | 	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
150 | 	cd %BUILDDIR%/latex
151 | 	make all-pdf
152 | 	cd %BUILDDIR%/..
153 | 	echo.
154 | 	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
155 | 	goto end
156 | )
157 | 
158 | if "%1" == "latexpdfja" (
159 | 	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
160 | 	cd %BUILDDIR%/latex
161 | 	make all-pdf-ja
162 | 	cd %BUILDDIR%/..
163 | 	echo.
164 | 	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
165 | 	goto end
166 | )
167 | 
168 | if "%1" == "text" (
169 | 	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
170 | 	if errorlevel 1 exit /b 1
171 | 	echo.
172 | 	echo.Build finished. The text files are in %BUILDDIR%/text.
173 | 	goto end
174 | )
175 | 
176 | if "%1" == "man" (
177 | 	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
178 | 	if errorlevel 1 exit /b 1
179 | 	echo.
180 | 	echo.Build finished. The manual pages are in %BUILDDIR%/man.
181 | 	goto end
182 | )
183 | 
184 | if "%1" == "texinfo" (
185 | 	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
186 | 	if errorlevel 1 exit /b 1
187 | 	echo.
188 | 	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
189 | 	goto end
190 | )
191 | 
192 | if "%1" == "gettext" (
193 | 	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
194 | 	if errorlevel 1 exit /b 1
195 | 	echo.
196 | 	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
197 | 	goto end
198 | )
199 | 
200 | if "%1" == "changes" (
201 | 	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
202 | 	if errorlevel 1 exit /b 1
203 | 	echo.
204 | 	echo.The overview file is in %BUILDDIR%/changes.
205 | 	goto end
206 | )
207 | 
208 | if "%1" == "linkcheck" (
209 | 	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
210 | 	if errorlevel 1 exit /b 1
211 | 	echo.
212 | 	echo.Link check complete; look for any errors in the above output ^
213 | or in %BUILDDIR%/linkcheck/output.txt.
214 | 	goto end
215 | )
216 | 
217 | if "%1" == "doctest" (
218 | 	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
219 | 	if errorlevel 1 exit /b 1
220 | 	echo.
221 | 	echo.Testing of doctests in the sources finished, look at the ^
222 | results in %BUILDDIR%/doctest/output.txt.
223 | 	goto end
224 | )
225 | 
226 | if "%1" == "xml" (
227 | 	%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
228 | 	if errorlevel 1 exit /b 1
229 | 	echo.
230 | 	echo.Build finished. The XML files are in %BUILDDIR%/xml.
231 | 	goto end
232 | )
233 | 
234 | if "%1" == "pseudoxml" (
235 | 	%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
236 | 	if errorlevel 1 exit /b 1
237 | 	echo.
238 | 	echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
239 | 	goto end
240 | )
241 | 
242 | :end
243 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | #
  3 | # Copyright (c) 2015-2021 Canonical, Ltd.
  4 | #
  5 | # This file is part of Talisker
  6 | # (see http://github.com/canonical-ols/talisker).
  7 | #
  8 | # Licensed to the Apache Software Foundation (ASF) under one
  9 | # or more contributor license agreements.  See the NOTICE file
 10 | # distributed with this work for additional information
 11 | # regarding copyright ownership.  The ASF licenses this file
 12 | # to you under the Apache License, Version 2.0 (the
 13 | # "License"); you may not use this file except in compliance
 14 | # with the License.  You may obtain a copy of the License at
 15 | #
 16 | #   http://www.apache.org/licenses/LICENSE-2.0
 17 | #
 18 | # Unless required by applicable law or agreed to in writing,
 19 | # software distributed under the License is distributed on an
 20 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 21 | # KIND, either express or implied.  See the License for the
 22 | # specific language governing permissions and limitations
 23 | # under the License.
 24 | #
 25 | #
 26 | # Note: this file is autogenerated from setup.cfg for older setuptools
 27 | #
 28 | try:
 29 |     from setuptools import setup
 30 | except ImportError:
 31 |     from distutils.core import setup
 32 | 
 33 | DESCRIPTION = '''
 34 | ===========================================
 35 | Talisker - an opinionated WSGI app platform
 36 | ===========================================
 37 | 
 38 | .. image:: https://img.shields.io/pypi/v/talisker.svg
 39 |     :target: https://pypi.python.org/pypi/talisker
 40 | 
 41 | .. image:: https://img.shields.io/travis/canonical-ols/talisker.svg
 42 |     :target: https://travis-ci.org/canonical-ols/talisker
 43 | 
 44 | .. image:: https://readthedocs.org/projects/talisker/badge/?version=latest
 45 |     :target: https://readthedocs.org/projects/talisker/?badge=latest
 46 |     :alt: Documentation Status
 47 | 
 48 | Talisker is an enhanced runtime for your WSGI application that aims to provide
 49 | a common operational platform for your python microservices.
 50 | 
 51 | It integrates with many standard python libraries to give you out-of-the-box
 52 | logging, metrics, error reporting, status urls and more.
 53 | 
 54 | Python version support
 55 | ----------------------
 56 | 
 57 | Talisker 0.20.0 was the last to support Python 2.7.
 58 | Talisker version >=0.21.0 only supports Python 3.x, as
 59 | they come with Ubuntu LTS releases.
 60 | 
 61 | Quick Start
 62 | -----------
 63 | 
 64 | Simply install Talisker with Gunicorn via pip::
 65 | 
 66 |     pip install talisker[gunicorn]
 67 | 
 68 | And then run your WSGI app with Talisker (as if it was regular gunicorn).::
 69 | 
 70 |     talisker.gunicorn app:wsgi -c config.py ...
 71 | 
 72 | This gives you 80% of the benefits of Talisker: structured logging, metrics,
 73 | sentry error handling, standardised status endpoints and more.
 74 | 
 75 | Note: right now, Talisker has extensive support for running with Gunicorn, with
 76 | more WSGI server support planned.
 77 | 
 78 | 
 79 | Elevator Pitch
 80 | --------------
 81 | 
 82 | Talisker integrates and configures standard python libraries into a single
 83 | tool, useful in both development and production. It provides:
 84 | 
 85 |   - structured logging for stdlib logging module (with grok filter)
 86 |   - gunicorn as a wsgi runner
 87 |   - request id tracing
 88 |   - standard status endpoints
 89 |   - statsd/prometheus metrics for incoming/outgoing http requests and more.
 90 |   - deep sentry integration
 91 | 
 92 | It also optionally supports the same level of logging/metrics/sentry
 93 | integration for:
 94 | 
 95 |  - celery workers
 96 |  - general python scripts, like cron jobs or management tasks.
 97 | 
 98 | Talisker is opinionated, and designed to be simple to use. As such, it is not
 99 | currently very configurable. However, PR's are very welcome!
100 | 
101 | For more information, see The Documentation, which should be found at:
102 | 
103 | https://talisker.readthedocs.io
104 | '''
105 | 
106 | setup(
107 |     author='Simon Davy',
108 |     author_email='simon.davy@canonical.com',
109 |     classifiers=[
110 |         'License :: OSI Approved :: Apache Software License',
111 |         'Development Status :: 4 - Beta',
112 |         'Intended Audience :: Developers',
113 |         'Natural Language :: English',
114 |         'Topic :: Internet :: WWW/HTTP :: WSGI',
115 |         'Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware',
116 |         'Topic :: System :: Logging',
117 |         'Programming Language :: Python :: 3.5',
118 |         'Programming Language :: Python :: 3.6',
119 |         'Programming Language :: Python :: 3.8',
120 |         'Programming Language :: Python :: 3.10',
121 |         'Programming Language :: Python :: 3.12',
122 |         'Programming Language :: Python :: Implementation :: CPython',
123 |     ],
124 |     description='A common WSGI stack',
125 |     entry_points=dict(
126 |         console_scripts=[
127 |             'talisker=talisker:run_gunicorn[gunicorn]',
128 |             'talisker.run=talisker:run',
129 |             'talisker.gunicorn=talisker:run_gunicorn[gunicorn]',
130 |             'talisker.gunicorn.eventlet=talisker:run_gunicorn_eventlet[gunicorn]',
131 |             'talisker.gunicorn.gevent=talisker:run_gunicorn_gevent[gunicorn]',
132 |             'talisker.celery=talisker:run_celery[celery]',
133 |             'talisker.help=talisker:run_help',
134 |         ],
135 |     ),
136 |     extras_require=dict(
137 |         asyncio=[
138 |             'aiocontextvars==0.2.2;python_version>="3.5.3" and python_version<"3.7"',
139 |         ],
140 |         celery=[
141 |             'celery~=4.4;python_version~="3.5.0"',
142 |             'celery>=4;python_version>"3.5"',
143 |         ],
144 |         dev=[
145 |             'logging_tree>=1.9',
146 |             'pygments>=2.11',
147 |             'psutil>=5.9',
148 |             'objgraph>=3.5',
149 |         ],
150 |         django=[
151 |             'django~=2.2;python_version~="3.5.0"',
152 |             'django<5;python_version>"3.5"',
153 |         ],
154 |         flask=[
155 |             'flask~=1.1;python_version~="3.5.0"',
156 |             'flask<3;python_version>"3.5"',
157 |             'blinker~=1.5;python_version~="3.5.0"',
158 |             'blinker<2;python_version>"3.5"',
159 |         ],
160 |         gevent=[
161 |             'gevent>=20.9.0',
162 |         ],
163 |         gunicorn=[
164 |             'gunicorn>=19.7.0;python_version>"3.6"',
165 |             'gunicorn>=19.7.0,<21.0;python_version>="3.5" and python_version<"3.8"',
166 |         ],
167 |         pg=[
168 |             'sqlparse>=0.4.2',
169 |             'psycopg2>=2.8,<3.0',
170 |         ],
171 |         prometheus=[
172 |             'prometheus-client~=0.7.0;python_version~="3.5.0"',
173 |             'prometheus-client<0.8;python_version>"3.5"',
174 |         ],
175 |         raven=[
176 |             'raven>=6.4.0',
177 |         ],
178 |     ),
179 |     include_package_data=True,
180 |     install_requires=[
181 |         'Werkzeug~=1.0;python_version~="3.5.0"',
182 |         'Werkzeug<3;python_version>="3.6"',
183 |         'statsd~=3.3;python_version~="3.5.0"',
184 |         'statsd<4;python_version>="3.6"',
185 |         'requests~=2.25;python_version~="3.5.0"',
186 |         'requests<3.0;python_version>"3.5"',
187 |         'contextvars~=2.4;python_version>="3.5" and python_version<"3.7"',
188 |     ],
189 |     keywords=[
190 |         'talisker',
191 |     ],
192 |     long_description=DESCRIPTION,
193 |     name='talisker',
194 |     package_data=dict(
195 |         talisker=[
196 |             'logstash/*',
197 |         ],
198 |     ),
199 |     package_dir=dict(
200 |         talisker='talisker',
201 |     ),
202 |     packages=[
203 |         'talisker',
204 |     ],
205 |     test_suite='tests',
206 |     url='https://github.com/canonical-ols/talisker',
207 |     version='0.21.5',
208 |     zip_safe=False,
209 | )
210 | 


--------------------------------------------------------------------------------
/tests/test_testing.py:
--------------------------------------------------------------------------------
  1 | #
  2 | # Copyright (c) 2015-2021 Canonical, Ltd.
  3 | #
  4 | # This file is part of Talisker
  5 | # (see http://github.com/canonical-ols/talisker).
  6 | #
  7 | # Licensed to the Apache Software Foundation (ASF) under one
  8 | # or more contributor license agreements.  See the NOTICE file
  9 | # distributed with this work for additional information
 10 | # regarding copyright ownership.  The ASF licenses this file
 11 | # to you under the Apache License, Version 2.0 (the
 12 | # "License"); you may not use this file except in compliance
 13 | # with the License.  You may obtain a copy of the License at
 14 | #
 15 | #   http://www.apache.org/licenses/LICENSE-2.0
 16 | #
 17 | # Unless required by applicable law or agreed to in writing,
 18 | # software distributed under the License is distributed on an
 19 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 20 | # KIND, either express or implied.  See the License for the
 21 | # specific language governing permissions and limitations
 22 | # under the License.
 23 | #
 24 | 
 25 | import logging
 26 | import sys
 27 | import textwrap
 28 | 
 29 | import pytest
 30 | import requests
 31 | 
 32 | import talisker.logs
 33 | from talisker import testing
 34 | 
 35 | 
 36 | def test_log_record_list():
 37 | 
 38 |     makeRecord = talisker.logs.StructuredLogger('test').makeRecord
 39 | 
 40 |     def record(name, level, msg, extra={}):
 41 |         return makeRecord(name, level, 'fn', 123, msg, None, None, extra=extra)
 42 | 
 43 |     r1 = record('root.log1', logging.INFO, 'foo msg')
 44 |     r2 = record('root.log1', logging.DEBUG, 'bar')
 45 |     r3 = record('root.log2', logging.WARNING, 'baz', extra={'a': 1})
 46 | 
 47 |     records = testing.LogRecordList()
 48 |     records.extend([r1, r2, r3])
 49 | 
 50 |     # name, ex
 51 |     assert records.filter(name='root.log1') == [r1, r2]
 52 |     assert records.filter(name='log1') == [r1, r2]
 53 |     assert records.filter(name='root') == [r1, r2, r3]
 54 | 
 55 |     # msg
 56 |     assert records.filter(msg='msg') == [r1]
 57 |     assert records.filter(msg='bar') == [r2]
 58 | 
 59 |     # level
 60 |     assert records.filter(name='log1', level=logging.INFO) == [r1]
 61 |     assert records.filter(name='log1', levelname='INFO') == [r1]
 62 |     assert records.filter(name='log1').filter(level=logging.DEBUG) == [r2]
 63 | 
 64 |     # extra
 65 |     assert records.filter(extra={'a': 1}) == [r3]
 66 |     assert records.filter(extra={'a': 2}) == []
 67 | 
 68 | 
 69 | def test_log_record_list_parse():
 70 |     handler = testing.TestHandler()
 71 |     handler.setFormatter(talisker.logs.StructuredFormatter())
 72 |     handler.setLevel(logging.NOTSET)
 73 |     logger = talisker.logs.StructuredLogger('test')
 74 |     logger.addHandler(handler)
 75 | 
 76 |     logger.info('msg 1')
 77 |     logger.info('msg 2 with extra', extra={'foo': 'barrrrr'})
 78 |     records = testing.LogRecordList.parse(handler.lines)
 79 | 
 80 |     records.assert_not_log(msg='not found')
 81 |     records.assert_log(msg='msg 1')
 82 |     records.assert_not_log(msg='msg 1', extra={'foo': 'baz'})
 83 |     records.assert_log(msg='msg 2')
 84 |     records.assert_log(msg='msg 2', extra={'foo': 'bar'})
 85 |     records.assert_not_log(msg='msg 2', extra={'foo': 'bar', 'baz': '1'})
 86 | 
 87 | 
 88 | @pytest.mark.skipif(not talisker.sentry.enabled, reason='raven not installed')
 89 | def test_test_context():
 90 | 
 91 |     assert testing.get_sentry_messages() == []
 92 | 
 93 |     logger = logging.getLogger('test_test_context')
 94 |     with testing.TestContext() as ctx:
 95 |         logger.info('foo', extra={'a': 1})
 96 |         logger.warning('bar', extra={'b': 2})
 97 |         talisker.statsd.get_client().timing('statsd', 3)
 98 |         talisker.sentry.get_client().capture(
 99 |             'Message',
100 |             message='test',
101 |             extra={
102 |                 'foo': 'bar'
103 |             },
104 |         )
105 | 
106 |     ctx.assert_log(
107 |         name=logger.name, msg='foo', level='info', extra={'a': 1})
108 |     ctx.assert_log(
109 |         name=logger.name, msg='bar', level='warning', extra={'b': 2})
110 | 
111 |     with pytest.raises(AssertionError) as exc:
112 |         ctx.assert_log(
113 |             name=logger.name, msg='XXX', level='info', extra={"baz": 1})
114 | 
115 |     assert str(exc.value) == textwrap.dedent("""
116 |         Could not find log out of {0} logs.
117 |         Search terms that could not be found:
118 |             msg=XXX
119 |             extra["baz"]=1
120 |     """).strip().format(len(ctx.logs), 'u' if sys.version_info[0] == 2 else '')
121 | 
122 |     with pytest.raises(AssertionError) as exc:
123 |         ctx.assert_not_log(name=logger.name, msg='foo', level='info')
124 | 
125 |     assert str(exc.value) == textwrap.dedent("""
126 |         Found log matching the following:
127 |             level={0}'info'
128 |             msg={0}'foo'
129 |             name='test_test_context'
130 |     """).strip().format('u' if sys.version_info[0] == 2 else '')
131 | 
132 |     assert ctx.statsd == ['statsd:3.000000|ms']
133 | 
134 |     # ensure there are not sentry messages left over
135 |     assert testing.get_sentry_messages() == []
136 |     assert len(ctx.sentry) == 1
137 |     assert ctx.sentry[0]['message'] == 'test'
138 |     # check that extra values have been decoded correctly
139 |     assert ctx.sentry[0]['extra']['foo'] == 'bar'
140 | 
141 | 
142 | def test_serverprocess_success():
143 |     server = testing.ServerProcess(['true'])
144 |     with server:
145 |         server.ps.wait()
146 |     assert server.finished
147 | 
148 | 
149 | def test_serverprocess_failure():
150 |     server = testing.ServerProcess(['false'])
151 |     with pytest.raises(testing.ServerProcessError):
152 |         with server:
153 |             while 1:
154 |                 server.check()
155 | 
156 | 
157 | def test_serverprocess_output(tmpdir):
158 |     script = tmpdir.join('script.sh')
159 |     script.write("for i in $(seq 20); do echo $i; done")
160 |     server = testing.ServerProcess(['bash', str(script)])
161 |     with server:
162 |         server.ps.wait()
163 |     assert server.output == [str(i) for i in range(1, 21)]
164 | 
165 | 
166 | def test_serverprocess_output_wait(tmpdir):
167 |     script = tmpdir.join('script.sh')
168 |     script.write("echo 1; echo 2; echo 'here'; read; echo 3; echo 4;")
169 |     server = testing.ServerProcess(['bash', str(script)])
170 |     with server:
171 |         server.wait_for_output('here', timeout=30)
172 |         assert server.output == ['1', '2', 'here']
173 |         server.ps.stdin.write('bar\n')
174 |         server.ps.wait()
175 | 
176 |     assert server.output == ['1', '2', 'here', '3', '4']
177 | 
178 | 
179 | def test_gunicornprocess_success():
180 |     try:
181 |         import gunicorn  # noqa
182 |     except ImportError:
183 |         pytest.skip('need gunicorn installed')
184 |     id = 'test-id'
185 |     ps = testing.GunicornProcess('tests.wsgi_app')
186 |     with ps:
187 |         r = requests.get(ps.url('/'), headers={'X-Request-Id': id})
188 |         assert r.status_code == 200
189 |     ps.log.assert_log(
190 |         msg='GET /',
191 |         extra={
192 |             'status': '200',
193 |             'method': 'GET',
194 |             'ip': '127.0.0.1',
195 |             'proto': 'HTTP/1.1',
196 |             'request_id': id,
197 |         }
198 |     )
199 | 
200 | 
201 | def test_gunicornprocess_bad_app():
202 |     gunicorn = testing.GunicornProcess('no_app_here')
203 |     with pytest.raises(testing.ServerProcessError):
204 |         with gunicorn:
205 |             pass
206 | 
207 | 
208 | def test_log_record_list_no_extra():
209 |     makeRecord = logging.Logger('test').makeRecord
210 | 
211 |     def record(name, level, msg):
212 |         return makeRecord(name, level, 'fn', 123, msg, None, None)
213 | 
214 |     r1 = record('root.log1', logging.WARNING, 'foo')
215 |     r2 = record('root.log2', logging.INFO, 'foo bar')
216 |     records = testing.LogRecordList()
217 |     records.extend([r1, r2])
218 | 
219 |     assert records.filter(name='root.log1') == [r1]
220 |     assert records.filter(msg='foo') == [r1, r2]
221 |     assert records.filter(level=logging.WARNING) == [r1]
222 |     assert records.filter(extra={'a': 1}) == []
223 | 


--------------------------------------------------------------------------------